{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use TD3 to Play Pendulum-v1\n",
    "\n",
    "PyTorch version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "import logging\n",
    "import itertools\n",
    "import copy\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import pandas as pd\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "torch.manual_seed(0)\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "00:00:00 [INFO] id: Pendulum-v1\n",
      "00:00:00 [INFO] entry_point: gym.envs.classic_control:PendulumEnv\n",
      "00:00:00 [INFO] reward_threshold: None\n",
      "00:00:00 [INFO] nondeterministic: False\n",
      "00:00:00 [INFO] max_episode_steps: 200\n",
      "00:00:00 [INFO] order_enforce: True\n",
      "00:00:00 [INFO] _kwargs: {}\n",
      "00:00:00 [INFO] _env_name: Pendulum\n",
      "00:00:00 [INFO] max_speed: 8\n",
      "00:00:00 [INFO] max_torque: 2.0\n",
      "00:00:00 [INFO] dt: 0.05\n",
      "00:00:00 [INFO] g: 10.0\n",
      "00:00:00 [INFO] m: 1.0\n",
      "00:00:00 [INFO] l: 1.0\n",
      "00:00:00 [INFO] viewer: None\n",
      "00:00:00 [INFO] action_space: Box([-2.], [2.], (1,), float32)\n",
      "00:00:00 [INFO] observation_space: Box([-1. -1. -8.], [1. 1. 8.], (3,), float32)\n",
      "00:00:00 [INFO] np_random: RandomState(MT19937)\n",
      "00:00:00 [INFO] spec: EnvSpec(Pendulum-v1)\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('Pendulum-v1')\n",
    "env.seed(0)\n",
    "for key in vars(env.spec):\n",
    "    logging.info('%s: %s', key, vars(env.spec)[key])\n",
    "for key in vars(env.unwrapped):\n",
    "    logging.info('%s: %s', key, vars(env.unwrapped)[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQNReplayer:\n",
    "    def __init__(self, capacity):\n",
    "        self.memory = pd.DataFrame(index=range(capacity),\n",
    "                columns=['observation', 'action', 'reward',\n",
    "                'next_observation', 'done'])\n",
    "        self.i = 0\n",
    "        self.count = 0\n",
    "        self.capacity = capacity\n",
    "\n",
    "    def store(self, *args):\n",
    "        self.memory.loc[self.i] = args\n",
    "        self.i = (self.i + 1) % self.capacity\n",
    "        self.count = min(self.count + 1, self.capacity)\n",
    "\n",
    "    def sample(self, size):\n",
    "        indices = np.random.choice(self.count, size=size)\n",
    "        return (np.stack(self.memory.loc[indices, field]) for field in\n",
    "                self.memory.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OrnsteinUhlenbeckProcess:\n",
    "    def __init__(self, x0):\n",
    "        self.x = x0\n",
    "\n",
    "    def __call__(self, mu=0., sigma=1., theta=.15, dt=.01):\n",
    "        n = np.random.normal(size=self.x.shape)\n",
    "        self.x += (theta * (mu - self.x) * dt + sigma * np.sqrt(dt) * n)\n",
    "        return self.x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TD3Agent:\n",
    "    def __init__(self, env):\n",
    "        state_dim = env.observation_space.shape[0]\n",
    "        self.action_dim = env.action_space.shape[0]\n",
    "        self.action_low = env.action_space.low[0]\n",
    "        self.action_high = env.action_space.high[0]\n",
    "\n",
    "        self.gamma = 0.99\n",
    "\n",
    "        self.replayer = DQNReplayer(20000)\n",
    "\n",
    "        self.actor_evaluate_net = self.build_net(\n",
    "                input_size=state_dim, hidden_sizes=[32, 64],\n",
    "                output_size=self.action_dim)\n",
    "        self.actor_optimizer = optim.Adam(self.actor_evaluate_net.parameters(), lr=0.001)\n",
    "        self.actor_target_net = copy.deepcopy(self.actor_evaluate_net)\n",
    "\n",
    "        self.critic0_evaluate_net = self.build_net(\n",
    "                input_size=state_dim+self.action_dim, hidden_sizes=[64, 128])\n",
    "        self.critic0_optimizer = optim.Adam(self.critic0_evaluate_net.parameters(), lr=0.001)\n",
    "        self.critic0_loss = nn.MSELoss()\n",
    "        self.critic0_target_net = copy.deepcopy(self.critic0_evaluate_net)\n",
    "\n",
    "        self.critic1_evaluate_net = self.build_net(\n",
    "                input_size=state_dim+self.action_dim, hidden_sizes=[64, 128])\n",
    "        self.critic1_optimizer = optim.Adam(self.critic1_evaluate_net.parameters(), lr=0.001)\n",
    "        self.critic1_loss = nn.MSELoss()\n",
    "        self.critic1_target_net = copy.deepcopy(self.critic1_evaluate_net)\n",
    "\n",
    "    def build_net(self, input_size, hidden_sizes, output_size=1,\n",
    "            output_activator=None):\n",
    "        layers = []\n",
    "        for input_size, output_size in zip(\n",
    "                [input_size,] + hidden_sizes, hidden_sizes + [output_size,]):\n",
    "            layers.append(nn.Linear(input_size, output_size))\n",
    "            layers.append(nn.ReLU())\n",
    "        layers = layers[:-1]\n",
    "        if output_activator:\n",
    "            layers.append(output_activator)\n",
    "        net = nn.Sequential(*layers)\n",
    "        return net\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory = []\n",
    "            self.noise = OrnsteinUhlenbeckProcess(np.zeros((self.action_dim,)))\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        state_tensor = torch.as_tensor(observation, dtype=torch.float).unsqueeze(0)\n",
    "        action_tensor = self.actor_evaluate_net(state_tensor)\n",
    "        action = action_tensor.detach().numpy()[0]\n",
    "\n",
    "        if self.mode == 'train':\n",
    "            # noisy action\n",
    "            action = (action + self.noise(sigma=0.1)).clip(self.action_low, self.action_high)\n",
    "\n",
    "            self.trajectory += [observation, reward, done, action]\n",
    "            if len(self.trajectory) >= 8:\n",
    "                state, _, _, act, next_state, reward, done, _ = self.trajectory[-8:]\n",
    "                self.replayer.store(state, act, reward, next_state, done)\n",
    "\n",
    "            # learn\n",
    "            if self.replayer.count >= 3000:\n",
    "                self.learn()\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        pass\n",
    "\n",
    "    def update_net(self, target_net, evaluate_net, learning_rate=0.005):\n",
    "        for target_param, evaluate_param in zip(\n",
    "                target_net.parameters(), evaluate_net.parameters()):\n",
    "            target_param.data.copy_(learning_rate * evaluate_param.data\n",
    "                    + (1 - learning_rate) * target_param.data)\n",
    "\n",
    "    def learn(self):\n",
    "        # replay\n",
    "        states, actions, rewards, next_states, dones = self.replayer.sample(64)\n",
    "        state_tensor = torch.as_tensor(states, dtype=torch.float)\n",
    "        action_tensor = torch.as_tensor(actions, dtype=torch.long)\n",
    "        reward_tensor = torch.as_tensor(rewards, dtype=torch.float)\n",
    "        next_state_tensor = torch.as_tensor(next_states, dtype=torch.float)\n",
    "        done_tensor = torch.as_tensor(dones, dtype=torch.float)\n",
    "\n",
    "        # learn critic\n",
    "        next_action_tensor = self.actor_target_net(next_state_tensor)\n",
    "        noise_tensor = (0.2 * torch.randn_like(action_tensor, dtype=torch.float))\n",
    "        noisy_next_action_tensor = (next_action_tensor + noise_tensor\n",
    "                    ).clamp(self.action_low, self.action_high)\n",
    "        next_state_action_tensor = torch.cat([next_state_tensor, noisy_next_action_tensor], 1)\n",
    "        next_q0_tensor = self.critic0_target_net(next_state_action_tensor).squeeze(1)\n",
    "        next_q1_tensor = self.critic1_target_net(next_state_action_tensor).squeeze(1)\n",
    "        next_q_tensor = torch.min(next_q0_tensor, next_q1_tensor)\n",
    "        critic_target_tensor = reward_tensor + (1. - done_tensor) * self.gamma * next_q_tensor\n",
    "        critic_target_tensor = critic_target_tensor.detach()\n",
    "\n",
    "        state_action_tensor = torch.cat([state_tensor, action_tensor], 1)\n",
    "        critic_pred0_tensor = self.critic0_evaluate_net(state_action_tensor).squeeze(1)\n",
    "        critic0_loss_tensor = self.critic0_loss(critic_pred0_tensor, critic_target_tensor)\n",
    "        self.critic0_optimizer.zero_grad()\n",
    "        critic0_loss_tensor.backward()\n",
    "        self.critic0_optimizer.step()\n",
    "\n",
    "        critic_pred1_tensor = self.critic1_evaluate_net(state_action_tensor).squeeze(1)\n",
    "        critic1_loss_tensor = self.critic1_loss(critic_pred1_tensor, critic_target_tensor)\n",
    "        self.critic1_optimizer.zero_grad()\n",
    "        critic1_loss_tensor.backward()\n",
    "        self.critic1_optimizer.step()\n",
    "\n",
    "        # learn actor\n",
    "        pred_action_tensor = self.actor_evaluate_net(state_tensor)\n",
    "        pred_action_tensor = pred_action_tensor.clamp(self.action_low, self.action_high)\n",
    "        pred_state_action_tensor = torch.cat([state_tensor, pred_action_tensor], 1)\n",
    "        critic_pred_tensor = self.critic0_evaluate_net(pred_state_action_tensor)\n",
    "        actor_loss_tensor = -critic_pred_tensor.mean()\n",
    "        self.actor_optimizer.zero_grad()\n",
    "        actor_loss_tensor.backward()\n",
    "        self.actor_optimizer.step()\n",
    "\n",
    "        self.update_net(self.critic0_target_net, self.critic0_evaluate_net)\n",
    "        self.update_net(self.critic1_target_net, self.critic1_evaluate_net)\n",
    "        self.update_net(self.actor_target_net, self.actor_evaluate_net)\n",
    "\n",
    "\n",
    "agent = TD3Agent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:50:06 [INFO] ==== train ====\n",
      "22:50:06 [DEBUG] train episode 0: reward = -1721.57, steps = 200\n",
      "22:50:06 [DEBUG] train episode 1: reward = -1070.98, steps = 200\n",
      "22:50:06 [DEBUG] train episode 2: reward = -1577.55, steps = 200\n",
      "22:50:06 [DEBUG] train episode 3: reward = -1060.81, steps = 200\n",
      "22:50:07 [DEBUG] train episode 4: reward = -1743.80, steps = 200\n",
      "22:50:07 [DEBUG] train episode 5: reward = -1762.58, steps = 200\n",
      "22:50:07 [DEBUG] train episode 6: reward = -951.33, steps = 200\n",
      "22:50:07 [DEBUG] train episode 7: reward = -1765.45, steps = 200\n",
      "22:50:07 [DEBUG] train episode 8: reward = -903.09, steps = 200\n",
      "22:50:07 [DEBUG] train episode 9: reward = -1765.03, steps = 200\n",
      "22:50:08 [DEBUG] train episode 10: reward = -1318.38, steps = 200\n",
      "22:50:08 [DEBUG] train episode 11: reward = -983.89, steps = 200\n",
      "22:50:08 [DEBUG] train episode 12: reward = -1515.06, steps = 200\n",
      "22:50:08 [DEBUG] train episode 13: reward = -1287.58, steps = 200\n",
      "22:50:08 [DEBUG] train episode 14: reward = -860.10, steps = 200\n",
      "22:50:32 [DEBUG] train episode 15: reward = -1504.97, steps = 200\n",
      "22:50:56 [DEBUG] train episode 16: reward = -1044.90, steps = 200\n",
      "22:51:21 [DEBUG] train episode 17: reward = -1386.82, steps = 200\n",
      "22:51:45 [DEBUG] train episode 18: reward = -1213.84, steps = 200\n",
      "22:52:10 [DEBUG] train episode 19: reward = -986.98, steps = 200\n",
      "22:52:35 [DEBUG] train episode 20: reward = -1322.80, steps = 200\n",
      "22:53:00 [DEBUG] train episode 21: reward = -1610.61, steps = 200\n",
      "22:53:25 [DEBUG] train episode 22: reward = -1593.37, steps = 200\n",
      "22:53:52 [DEBUG] train episode 23: reward = -1299.74, steps = 200\n",
      "22:54:19 [DEBUG] train episode 24: reward = -930.12, steps = 200\n",
      "22:54:46 [DEBUG] train episode 25: reward = -1632.12, steps = 200\n",
      "22:55:11 [DEBUG] train episode 26: reward = -1203.33, steps = 200\n",
      "22:55:36 [DEBUG] train episode 27: reward = -982.27, steps = 200\n",
      "22:56:00 [DEBUG] train episode 28: reward = -948.41, steps = 200\n",
      "22:56:25 [DEBUG] train episode 29: reward = -1649.39, steps = 200\n",
      "22:56:50 [DEBUG] train episode 30: reward = -1100.71, steps = 200\n",
      "22:57:15 [DEBUG] train episode 31: reward = -1114.19, steps = 200\n",
      "22:57:40 [DEBUG] train episode 32: reward = -1047.40, steps = 200\n",
      "22:58:05 [DEBUG] train episode 33: reward = -999.32, steps = 200\n",
      "22:58:29 [DEBUG] train episode 34: reward = -1104.40, steps = 200\n",
      "22:58:55 [DEBUG] train episode 35: reward = -1028.04, steps = 200\n",
      "22:59:20 [DEBUG] train episode 36: reward = -1766.86, steps = 200\n",
      "22:59:45 [DEBUG] train episode 37: reward = -1178.15, steps = 200\n",
      "23:00:13 [DEBUG] train episode 38: reward = -961.91, steps = 200\n",
      "23:00:46 [DEBUG] train episode 39: reward = -1148.73, steps = 200\n",
      "23:01:17 [DEBUG] train episode 40: reward = -1765.17, steps = 200\n",
      "23:01:50 [DEBUG] train episode 41: reward = -1606.13, steps = 200\n",
      "23:02:16 [DEBUG] train episode 42: reward = -1164.91, steps = 200\n",
      "23:02:42 [DEBUG] train episode 43: reward = -1044.43, steps = 200\n",
      "23:03:12 [DEBUG] train episode 44: reward = -1202.69, steps = 200\n",
      "23:03:42 [DEBUG] train episode 45: reward = -1178.19, steps = 200\n",
      "23:04:07 [DEBUG] train episode 46: reward = -878.43, steps = 200\n",
      "23:04:34 [DEBUG] train episode 47: reward = -773.48, steps = 200\n",
      "23:05:03 [DEBUG] train episode 48: reward = -925.66, steps = 200\n",
      "23:05:29 [DEBUG] train episode 49: reward = -912.90, steps = 200\n",
      "23:05:56 [DEBUG] train episode 50: reward = -1660.76, steps = 200\n",
      "23:06:23 [DEBUG] train episode 51: reward = -760.19, steps = 200\n",
      "23:06:50 [DEBUG] train episode 52: reward = -904.53, steps = 200\n",
      "23:07:16 [DEBUG] train episode 53: reward = -723.88, steps = 200\n",
      "23:07:42 [DEBUG] train episode 54: reward = -893.00, steps = 200\n",
      "23:08:07 [DEBUG] train episode 55: reward = -1139.41, steps = 200\n",
      "23:08:33 [DEBUG] train episode 56: reward = -1039.83, steps = 200\n",
      "23:09:00 [DEBUG] train episode 57: reward = -1011.66, steps = 200\n",
      "23:09:26 [DEBUG] train episode 58: reward = -713.38, steps = 200\n",
      "23:09:54 [DEBUG] train episode 59: reward = -1010.66, steps = 200\n",
      "23:10:23 [DEBUG] train episode 60: reward = -898.30, steps = 200\n",
      "23:10:51 [DEBUG] train episode 61: reward = -883.27, steps = 200\n",
      "23:11:21 [DEBUG] train episode 62: reward = -792.54, steps = 200\n",
      "23:11:56 [DEBUG] train episode 63: reward = -754.38, steps = 200\n",
      "23:12:39 [DEBUG] train episode 64: reward = -883.61, steps = 200\n",
      "23:13:24 [DEBUG] train episode 65: reward = -898.71, steps = 200\n",
      "23:14:12 [DEBUG] train episode 66: reward = -908.78, steps = 200\n",
      "23:14:58 [DEBUG] train episode 67: reward = -891.53, steps = 200\n",
      "23:15:43 [DEBUG] train episode 68: reward = -992.78, steps = 200\n",
      "23:16:30 [DEBUG] train episode 69: reward = -895.59, steps = 200\n",
      "23:17:16 [DEBUG] train episode 70: reward = -904.98, steps = 200\n",
      "23:18:01 [DEBUG] train episode 71: reward = -1001.45, steps = 200\n",
      "23:18:46 [DEBUG] train episode 72: reward = -1126.34, steps = 200\n",
      "23:19:31 [DEBUG] train episode 73: reward = -1043.43, steps = 200\n",
      "23:20:17 [DEBUG] train episode 74: reward = -1067.01, steps = 200\n",
      "23:21:04 [DEBUG] train episode 75: reward = -1042.23, steps = 200\n",
      "23:21:49 [DEBUG] train episode 76: reward = -1050.97, steps = 200\n",
      "23:22:35 [DEBUG] train episode 77: reward = -1127.45, steps = 200\n",
      "23:23:21 [DEBUG] train episode 78: reward = -1145.52, steps = 200\n",
      "23:24:10 [DEBUG] train episode 79: reward = -1035.29, steps = 200\n",
      "23:24:55 [DEBUG] train episode 80: reward = -1131.83, steps = 200\n",
      "23:25:41 [DEBUG] train episode 81: reward = -1049.22, steps = 200\n",
      "23:26:30 [DEBUG] train episode 82: reward = -964.45, steps = 200\n",
      "23:27:15 [DEBUG] train episode 83: reward = -890.96, steps = 200\n",
      "23:28:01 [DEBUG] train episode 84: reward = -828.35, steps = 200\n",
      "23:28:47 [DEBUG] train episode 85: reward = -700.21, steps = 200\n",
      "23:29:33 [DEBUG] train episode 86: reward = -839.80, steps = 200\n",
      "23:30:20 [DEBUG] train episode 87: reward = -747.70, steps = 200\n",
      "23:31:09 [DEBUG] train episode 88: reward = -747.97, steps = 200\n",
      "23:31:54 [DEBUG] train episode 89: reward = -752.49, steps = 200\n",
      "23:32:40 [DEBUG] train episode 90: reward = -746.77, steps = 200\n",
      "23:33:25 [DEBUG] train episode 91: reward = -861.00, steps = 200\n",
      "23:34:10 [DEBUG] train episode 92: reward = -780.87, steps = 200\n",
      "23:34:56 [DEBUG] train episode 93: reward = -639.99, steps = 200\n",
      "23:35:41 [DEBUG] train episode 94: reward = -641.45, steps = 200\n",
      "23:36:28 [DEBUG] train episode 95: reward = -817.88, steps = 200\n",
      "23:37:15 [DEBUG] train episode 96: reward = -629.48, steps = 200\n",
      "23:38:02 [DEBUG] train episode 97: reward = -622.91, steps = 200\n",
      "23:38:51 [DEBUG] train episode 98: reward = -640.05, steps = 200\n",
      "23:39:37 [DEBUG] train episode 99: reward = -604.30, steps = 200\n",
      "23:40:24 [DEBUG] train episode 100: reward = -774.47, steps = 200\n",
      "23:41:10 [DEBUG] train episode 101: reward = -494.49, steps = 200\n",
      "23:41:55 [DEBUG] train episode 102: reward = -884.44, steps = 200\n",
      "23:42:48 [DEBUG] train episode 103: reward = -635.08, steps = 200\n",
      "23:43:43 [DEBUG] train episode 104: reward = -882.61, steps = 200\n",
      "23:44:33 [DEBUG] train episode 105: reward = -383.83, steps = 200\n",
      "23:45:18 [DEBUG] train episode 106: reward = -755.98, steps = 200\n",
      "23:46:09 [DEBUG] train episode 107: reward = -877.50, steps = 200\n",
      "23:46:57 [DEBUG] train episode 108: reward = -509.54, steps = 200\n",
      "23:47:55 [DEBUG] train episode 109: reward = -1031.94, steps = 200\n",
      "23:48:49 [DEBUG] train episode 110: reward = -759.50, steps = 200\n",
      "23:49:39 [DEBUG] train episode 111: reward = -752.53, steps = 200\n",
      "23:50:31 [DEBUG] train episode 112: reward = -957.64, steps = 200\n",
      "23:51:19 [DEBUG] train episode 113: reward = -882.54, steps = 200\n",
      "23:52:10 [DEBUG] train episode 114: reward = -870.90, steps = 200\n",
      "23:52:59 [DEBUG] train episode 115: reward = -982.60, steps = 200\n",
      "23:53:51 [DEBUG] train episode 116: reward = -963.98, steps = 200\n",
      "23:54:38 [DEBUG] train episode 117: reward = -867.98, steps = 200\n",
      "23:55:24 [DEBUG] train episode 118: reward = -868.38, steps = 200\n",
      "23:56:09 [DEBUG] train episode 119: reward = -769.28, steps = 200\n",
      "23:56:54 [DEBUG] train episode 120: reward = -922.79, steps = 200\n",
      "23:57:40 [DEBUG] train episode 121: reward = -847.18, steps = 200\n",
      "23:58:26 [DEBUG] train episode 122: reward = -866.54, steps = 200\n",
      "23:59:12 [DEBUG] train episode 123: reward = -904.55, steps = 200\n",
      "23:59:57 [DEBUG] train episode 124: reward = -513.19, steps = 200\n",
      "00:00:43 [DEBUG] train episode 125: reward = -721.57, steps = 200\n",
      "00:01:29 [DEBUG] train episode 126: reward = -638.88, steps = 200\n",
      "00:02:15 [DEBUG] train episode 127: reward = -522.03, steps = 200\n",
      "00:03:01 [DEBUG] train episode 128: reward = -516.54, steps = 200\n",
      "00:03:47 [DEBUG] train episode 129: reward = -614.31, steps = 200\n",
      "00:04:33 [DEBUG] train episode 130: reward = -510.68, steps = 200\n",
      "00:05:18 [DEBUG] train episode 131: reward = -627.74, steps = 200\n",
      "00:06:04 [DEBUG] train episode 132: reward = -484.04, steps = 200\n",
      "00:06:46 [DEBUG] train episode 133: reward = -498.52, steps = 200\n",
      "00:07:32 [DEBUG] train episode 134: reward = -622.09, steps = 200\n",
      "00:08:18 [DEBUG] train episode 135: reward = -673.45, steps = 200\n",
      "00:09:04 [DEBUG] train episode 136: reward = -419.64, steps = 200\n",
      "00:09:49 [DEBUG] train episode 137: reward = -598.43, steps = 200\n",
      "00:10:35 [DEBUG] train episode 138: reward = -383.04, steps = 200\n",
      "00:11:20 [DEBUG] train episode 139: reward = -493.48, steps = 200\n",
      "00:12:06 [DEBUG] train episode 140: reward = -134.65, steps = 200\n",
      "00:12:51 [DEBUG] train episode 141: reward = -368.43, steps = 200\n",
      "00:13:37 [DEBUG] train episode 142: reward = -551.87, steps = 200\n",
      "00:14:23 [DEBUG] train episode 143: reward = -131.40, steps = 200\n",
      "00:15:08 [DEBUG] train episode 144: reward = -262.17, steps = 200\n",
      "00:15:53 [DEBUG] train episode 145: reward = -403.52, steps = 200\n",
      "00:16:39 [DEBUG] train episode 146: reward = -375.08, steps = 200\n",
      "00:17:23 [DEBUG] train episode 147: reward = -506.14, steps = 200\n",
      "00:18:09 [DEBUG] train episode 148: reward = -512.21, steps = 200\n",
      "00:18:53 [DEBUG] train episode 149: reward = -716.76, steps = 200\n",
      "00:19:39 [DEBUG] train episode 150: reward = -722.97, steps = 200\n",
      "00:20:24 [DEBUG] train episode 151: reward = -623.30, steps = 200\n",
      "00:21:10 [DEBUG] train episode 152: reward = -298.58, steps = 200\n",
      "00:21:54 [DEBUG] train episode 153: reward = -498.90, steps = 200\n",
      "00:22:40 [DEBUG] train episode 154: reward = -506.79, steps = 200\n",
      "00:23:25 [DEBUG] train episode 155: reward = -627.91, steps = 200\n",
      "00:24:10 [DEBUG] train episode 156: reward = -273.54, steps = 200\n",
      "00:24:55 [DEBUG] train episode 157: reward = -514.32, steps = 200\n",
      "00:25:40 [DEBUG] train episode 158: reward = -265.99, steps = 200\n",
      "00:26:27 [DEBUG] train episode 159: reward = -140.16, steps = 200\n",
      "00:27:15 [DEBUG] train episode 160: reward = -366.77, steps = 200\n",
      "00:28:00 [DEBUG] train episode 161: reward = -135.96, steps = 200\n",
      "00:28:45 [DEBUG] train episode 162: reward = -449.60, steps = 200\n",
      "00:29:31 [DEBUG] train episode 163: reward = -257.36, steps = 200\n",
      "00:30:16 [DEBUG] train episode 164: reward = -263.04, steps = 200\n",
      "00:31:01 [DEBUG] train episode 165: reward = -144.86, steps = 200\n",
      "00:31:47 [DEBUG] train episode 166: reward = -384.65, steps = 200\n",
      "00:32:37 [DEBUG] train episode 167: reward = -709.58, steps = 200\n",
      "00:33:23 [DEBUG] train episode 168: reward = -504.85, steps = 200\n",
      "00:34:07 [DEBUG] train episode 169: reward = -422.78, steps = 200\n",
      "00:34:52 [DEBUG] train episode 170: reward = -810.01, steps = 200\n",
      "00:35:35 [DEBUG] train episode 171: reward = -564.94, steps = 200\n",
      "00:36:20 [DEBUG] train episode 172: reward = -388.60, steps = 200\n",
      "00:37:08 [DEBUG] train episode 173: reward = -668.61, steps = 200\n",
      "00:37:54 [DEBUG] train episode 174: reward = -501.38, steps = 200\n",
      "00:38:39 [DEBUG] train episode 175: reward = -463.29, steps = 200\n",
      "00:39:24 [DEBUG] train episode 176: reward = -406.66, steps = 200\n",
      "00:40:06 [DEBUG] train episode 177: reward = -604.76, steps = 200\n",
      "00:40:51 [DEBUG] train episode 178: reward = -253.91, steps = 200\n",
      "00:41:36 [DEBUG] train episode 179: reward = -486.96, steps = 200\n",
      "00:42:21 [DEBUG] train episode 180: reward = -576.04, steps = 200\n",
      "00:43:06 [DEBUG] train episode 181: reward = -375.20, steps = 200\n",
      "00:43:51 [DEBUG] train episode 182: reward = -493.18, steps = 200\n",
      "00:44:36 [DEBUG] train episode 183: reward = -373.14, steps = 200\n",
      "00:45:21 [DEBUG] train episode 184: reward = -234.01, steps = 200\n",
      "00:46:06 [DEBUG] train episode 185: reward = -127.60, steps = 200\n",
      "00:46:51 [DEBUG] train episode 186: reward = -253.05, steps = 200\n",
      "00:47:35 [DEBUG] train episode 187: reward = -7.27, steps = 200\n",
      "00:48:19 [DEBUG] train episode 188: reward = -361.74, steps = 200\n",
      "00:49:04 [DEBUG] train episode 189: reward = -508.00, steps = 200\n",
      "00:49:49 [DEBUG] train episode 190: reward = -246.49, steps = 200\n",
      "00:50:35 [DEBUG] train episode 191: reward = -484.12, steps = 200\n",
      "00:51:20 [DEBUG] train episode 192: reward = -371.06, steps = 200\n",
      "00:52:05 [DEBUG] train episode 193: reward = -575.35, steps = 200\n",
      "00:52:50 [DEBUG] train episode 194: reward = -255.27, steps = 200\n",
      "00:53:35 [DEBUG] train episode 195: reward = -371.99, steps = 200\n",
      "00:54:19 [DEBUG] train episode 196: reward = -372.93, steps = 200\n",
      "00:55:04 [DEBUG] train episode 197: reward = -363.40, steps = 200\n",
      "00:55:49 [DEBUG] train episode 198: reward = -501.59, steps = 200\n",
      "00:56:34 [DEBUG] train episode 199: reward = -129.10, steps = 200\n",
      "00:57:19 [DEBUG] train episode 200: reward = -503.00, steps = 200\n",
      "00:57:54 [DEBUG] train episode 201: reward = -120.43, steps = 200\n",
      "00:58:30 [DEBUG] train episode 202: reward = -254.81, steps = 200\n",
      "00:59:15 [DEBUG] train episode 203: reward = -131.72, steps = 200\n",
      "00:59:58 [DEBUG] train episode 204: reward = -126.23, steps = 200\n",
      "01:00:43 [DEBUG] train episode 205: reward = -125.94, steps = 200\n",
      "01:01:28 [DEBUG] train episode 206: reward = -125.51, steps = 200\n",
      "01:02:13 [DEBUG] train episode 207: reward = -255.84, steps = 200\n",
      "01:02:58 [DEBUG] train episode 208: reward = -246.80, steps = 200\n",
      "01:03:42 [DEBUG] train episode 209: reward = -3.75, steps = 200\n",
      "01:04:26 [DEBUG] train episode 210: reward = -124.18, steps = 200\n",
      "01:05:11 [DEBUG] train episode 211: reward = -374.76, steps = 200\n",
      "01:05:55 [DEBUG] train episode 212: reward = -137.36, steps = 200\n",
      "01:06:40 [DEBUG] train episode 213: reward = -3.13, steps = 200\n",
      "01:07:25 [DEBUG] train episode 214: reward = -509.80, steps = 200\n",
      "01:08:11 [DEBUG] train episode 215: reward = -132.43, steps = 200\n",
      "01:08:56 [DEBUG] train episode 216: reward = -132.30, steps = 200\n",
      "01:09:41 [DEBUG] train episode 217: reward = -123.27, steps = 200\n",
      "01:10:26 [DEBUG] train episode 218: reward = -248.48, steps = 200\n",
      "01:11:11 [DEBUG] train episode 219: reward = -254.86, steps = 200\n",
      "01:11:55 [DEBUG] train episode 220: reward = -123.96, steps = 200\n",
      "01:12:40 [DEBUG] train episode 221: reward = -125.79, steps = 200\n",
      "01:13:25 [DEBUG] train episode 222: reward = -3.26, steps = 200\n",
      "01:14:10 [DEBUG] train episode 223: reward = -555.67, steps = 200\n",
      "01:14:55 [DEBUG] train episode 224: reward = -487.66, steps = 200\n",
      "01:15:40 [DEBUG] train episode 225: reward = -633.67, steps = 200\n",
      "01:16:25 [DEBUG] train episode 226: reward = -382.08, steps = 200\n",
      "01:17:09 [DEBUG] train episode 227: reward = -498.68, steps = 200\n",
      "01:17:54 [DEBUG] train episode 228: reward = -248.48, steps = 200\n",
      "01:18:38 [DEBUG] train episode 229: reward = -261.38, steps = 200\n",
      "01:19:22 [DEBUG] train episode 230: reward = -373.96, steps = 200\n",
      "01:20:08 [DEBUG] train episode 231: reward = -627.17, steps = 200\n",
      "01:20:53 [DEBUG] train episode 232: reward = -760.25, steps = 200\n",
      "01:21:37 [DEBUG] train episode 233: reward = -138.81, steps = 200\n",
      "01:22:22 [DEBUG] train episode 234: reward = -253.40, steps = 200\n",
      "01:23:08 [DEBUG] train episode 235: reward = -351.31, steps = 200\n",
      "01:23:51 [DEBUG] train episode 236: reward = -126.64, steps = 200\n",
      "01:24:37 [DEBUG] train episode 237: reward = -130.42, steps = 200\n",
      "01:25:21 [DEBUG] train episode 238: reward = -375.13, steps = 200\n",
      "01:26:06 [DEBUG] train episode 239: reward = -371.86, steps = 200\n",
      "01:26:54 [DEBUG] train episode 240: reward = -129.06, steps = 200\n",
      "01:27:40 [DEBUG] train episode 241: reward = -341.52, steps = 200\n",
      "01:28:25 [DEBUG] train episode 242: reward = -131.11, steps = 200\n",
      "01:29:10 [DEBUG] train episode 243: reward = -259.53, steps = 200\n",
      "01:29:56 [DEBUG] train episode 244: reward = -384.46, steps = 200\n",
      "01:30:41 [DEBUG] train episode 245: reward = -140.34, steps = 200\n",
      "01:31:27 [DEBUG] train episode 246: reward = -265.71, steps = 200\n",
      "01:32:12 [DEBUG] train episode 247: reward = -239.26, steps = 200\n",
      "01:32:57 [DEBUG] train episode 248: reward = -317.49, steps = 200\n",
      "01:33:42 [DEBUG] train episode 249: reward = -129.42, steps = 200\n",
      "01:34:27 [DEBUG] train episode 250: reward = -133.52, steps = 200\n",
      "01:35:12 [DEBUG] train episode 251: reward = -249.69, steps = 200\n",
      "01:35:57 [DEBUG] train episode 252: reward = -134.03, steps = 200\n",
      "01:36:42 [DEBUG] train episode 253: reward = -293.11, steps = 200\n",
      "01:37:28 [DEBUG] train episode 254: reward = -128.80, steps = 200\n",
      "01:38:13 [DEBUG] train episode 255: reward = -240.41, steps = 200\n",
      "01:38:57 [DEBUG] train episode 256: reward = -120.32, steps = 200\n",
      "01:39:42 [DEBUG] train episode 257: reward = -134.86, steps = 200\n",
      "01:40:27 [DEBUG] train episode 258: reward = -133.42, steps = 200\n",
      "01:41:12 [DEBUG] train episode 259: reward = -248.89, steps = 200\n",
      "01:41:57 [DEBUG] train episode 260: reward = -131.94, steps = 200\n",
      "01:42:43 [DEBUG] train episode 261: reward = -393.44, steps = 200\n",
      "01:43:27 [DEBUG] train episode 262: reward = -129.26, steps = 200\n",
      "01:44:13 [DEBUG] train episode 263: reward = -7.18, steps = 200\n",
      "01:44:58 [DEBUG] train episode 264: reward = -251.55, steps = 200\n",
      "01:45:42 [DEBUG] train episode 265: reward = -245.81, steps = 200\n",
      "01:46:27 [DEBUG] train episode 266: reward = -247.89, steps = 200\n",
      "01:47:12 [DEBUG] train episode 267: reward = -131.22, steps = 200\n",
      "01:47:56 [DEBUG] train episode 268: reward = -134.16, steps = 200\n",
      "01:48:41 [DEBUG] train episode 269: reward = -137.25, steps = 200\n",
      "01:49:25 [DEBUG] train episode 270: reward = -356.23, steps = 200\n",
      "01:50:10 [DEBUG] train episode 271: reward = -8.03, steps = 200\n",
      "01:50:54 [DEBUG] train episode 272: reward = -137.97, steps = 200\n",
      "01:51:40 [DEBUG] train episode 273: reward = -122.40, steps = 200\n",
      "01:52:26 [DEBUG] train episode 274: reward = -252.92, steps = 200\n",
      "01:53:11 [DEBUG] train episode 275: reward = -137.42, steps = 200\n",
      "01:53:56 [DEBUG] train episode 276: reward = -122.92, steps = 200\n",
      "01:54:41 [DEBUG] train episode 277: reward = -255.63, steps = 200\n",
      "01:55:27 [DEBUG] train episode 278: reward = -648.33, steps = 200\n",
      "01:56:11 [DEBUG] train episode 279: reward = -124.46, steps = 200\n",
      "01:56:57 [DEBUG] train episode 280: reward = -136.56, steps = 200\n",
      "01:57:42 [DEBUG] train episode 281: reward = -259.56, steps = 200\n",
      "01:58:27 [DEBUG] train episode 282: reward = -390.69, steps = 200\n",
      "01:59:11 [DEBUG] train episode 283: reward = -247.74, steps = 200\n",
      "01:59:56 [DEBUG] train episode 284: reward = -128.01, steps = 200\n",
      "02:00:41 [DEBUG] train episode 285: reward = -142.94, steps = 200\n",
      "02:01:26 [DEBUG] train episode 286: reward = -133.63, steps = 200\n",
      "02:02:10 [DEBUG] train episode 287: reward = -364.54, steps = 200\n",
      "02:02:55 [DEBUG] train episode 288: reward = -140.83, steps = 200\n",
      "02:03:39 [DEBUG] train episode 289: reward = -132.56, steps = 200\n",
      "02:04:23 [DEBUG] train episode 290: reward = -132.94, steps = 200\n",
      "02:05:08 [DEBUG] train episode 291: reward = -350.31, steps = 200\n",
      "02:05:53 [DEBUG] train episode 292: reward = -620.37, steps = 200\n",
      "02:06:38 [DEBUG] train episode 293: reward = -10.93, steps = 200\n",
      "02:07:23 [DEBUG] train episode 294: reward = -388.36, steps = 200\n",
      "02:08:08 [DEBUG] train episode 295: reward = -383.58, steps = 200\n",
      "02:08:53 [DEBUG] train episode 296: reward = -26.43, steps = 200\n",
      "02:09:38 [DEBUG] train episode 297: reward = -380.80, steps = 200\n",
      "02:10:24 [DEBUG] train episode 298: reward = -380.96, steps = 200\n",
      "02:11:08 [DEBUG] train episode 299: reward = -131.45, steps = 200\n",
      "02:11:53 [DEBUG] train episode 300: reward = -133.48, steps = 200\n",
      "02:12:39 [DEBUG] train episode 301: reward = -379.10, steps = 200\n",
      "02:13:24 [DEBUG] train episode 302: reward = -254.04, steps = 200\n",
      "02:14:09 [DEBUG] train episode 303: reward = -139.14, steps = 200\n",
      "02:14:53 [DEBUG] train episode 304: reward = -256.20, steps = 200\n",
      "02:15:38 [DEBUG] train episode 305: reward = -129.71, steps = 200\n",
      "02:16:23 [DEBUG] train episode 306: reward = -138.65, steps = 200\n",
      "02:17:08 [DEBUG] train episode 307: reward = -11.28, steps = 200\n",
      "02:17:52 [DEBUG] train episode 308: reward = -363.82, steps = 200\n",
      "02:18:38 [DEBUG] train episode 309: reward = -133.72, steps = 200\n",
      "02:19:22 [DEBUG] train episode 310: reward = -386.97, steps = 200\n",
      "02:20:07 [DEBUG] train episode 311: reward = -129.97, steps = 200\n",
      "02:20:53 [DEBUG] train episode 312: reward = -238.23, steps = 200\n",
      "02:21:37 [DEBUG] train episode 313: reward = -263.25, steps = 200\n",
      "02:22:23 [DEBUG] train episode 314: reward = -252.38, steps = 200\n",
      "02:23:08 [DEBUG] train episode 315: reward = -141.11, steps = 200\n",
      "02:23:54 [DEBUG] train episode 316: reward = -131.52, steps = 200\n",
      "02:24:39 [DEBUG] train episode 317: reward = -141.00, steps = 200\n",
      "02:25:24 [DEBUG] train episode 318: reward = -144.32, steps = 200\n",
      "02:26:09 [DEBUG] train episode 319: reward = -354.82, steps = 200\n",
      "02:26:56 [DEBUG] train episode 320: reward = -146.45, steps = 200\n",
      "02:27:41 [DEBUG] train episode 321: reward = -142.70, steps = 200\n",
      "02:28:26 [DEBUG] train episode 322: reward = -149.18, steps = 200\n",
      "02:29:10 [DEBUG] train episode 323: reward = -260.30, steps = 200\n",
      "02:29:55 [DEBUG] train episode 324: reward = -146.20, steps = 200\n",
      "02:30:40 [DEBUG] train episode 325: reward = -147.51, steps = 200\n",
      "02:31:25 [DEBUG] train episode 326: reward = -351.62, steps = 200\n",
      "02:32:10 [DEBUG] train episode 327: reward = -26.76, steps = 200\n",
      "02:32:55 [DEBUG] train episode 328: reward = -152.73, steps = 200\n",
      "02:33:39 [DEBUG] train episode 329: reward = -277.68, steps = 200\n",
      "02:34:24 [DEBUG] train episode 330: reward = -272.10, steps = 200\n",
      "02:35:08 [DEBUG] train episode 331: reward = -511.88, steps = 200\n",
      "02:35:53 [DEBUG] train episode 332: reward = -505.30, steps = 200\n",
      "02:36:38 [DEBUG] train episode 333: reward = -377.27, steps = 200\n",
      "02:37:23 [DEBUG] train episode 334: reward = -124.82, steps = 200\n",
      "02:38:09 [DEBUG] train episode 335: reward = -509.91, steps = 200\n",
      "02:38:54 [DEBUG] train episode 336: reward = -500.62, steps = 200\n",
      "02:39:27 [DEBUG] train episode 337: reward = -378.70, steps = 200\n",
      "02:40:11 [DEBUG] train episode 338: reward = -501.04, steps = 200\n",
      "02:40:56 [DEBUG] train episode 339: reward = -388.74, steps = 200\n",
      "02:41:41 [DEBUG] train episode 340: reward = -484.26, steps = 200\n",
      "02:42:15 [DEBUG] train episode 341: reward = -386.19, steps = 200\n",
      "02:43:00 [DEBUG] train episode 342: reward = -486.55, steps = 200\n",
      "02:43:45 [DEBUG] train episode 343: reward = -488.01, steps = 200\n",
      "02:44:29 [DEBUG] train episode 344: reward = -250.64, steps = 200\n",
      "02:45:14 [DEBUG] train episode 345: reward = -387.91, steps = 200\n",
      "02:45:58 [DEBUG] train episode 346: reward = -510.38, steps = 200\n",
      "02:46:42 [DEBUG] train episode 347: reward = -483.42, steps = 200\n",
      "02:47:27 [DEBUG] train episode 348: reward = -488.33, steps = 200\n",
      "02:48:11 [DEBUG] train episode 349: reward = -365.64, steps = 200\n",
      "02:48:56 [DEBUG] train episode 350: reward = -263.37, steps = 200\n",
      "02:49:41 [DEBUG] train episode 351: reward = -506.30, steps = 200\n",
      "02:50:26 [DEBUG] train episode 352: reward = -384.13, steps = 200\n",
      "02:51:10 [DEBUG] train episode 353: reward = -503.64, steps = 200\n",
      "02:51:55 [DEBUG] train episode 354: reward = -378.24, steps = 200\n",
      "02:52:41 [DEBUG] train episode 355: reward = -372.56, steps = 200\n",
      "02:53:26 [DEBUG] train episode 356: reward = -464.91, steps = 200\n",
      "02:54:12 [DEBUG] train episode 357: reward = -334.28, steps = 200\n",
      "02:54:57 [DEBUG] train episode 358: reward = -138.82, steps = 200\n",
      "02:55:42 [DEBUG] train episode 359: reward = -383.96, steps = 200\n",
      "02:56:24 [DEBUG] train episode 360: reward = -491.77, steps = 200\n",
      "02:57:00 [DEBUG] train episode 361: reward = -145.44, steps = 200\n",
      "02:57:36 [DEBUG] train episode 362: reward = -172.86, steps = 200\n",
      "02:58:12 [DEBUG] train episode 363: reward = -373.13, steps = 200\n",
      "02:58:48 [DEBUG] train episode 364: reward = -263.49, steps = 200\n",
      "02:59:23 [DEBUG] train episode 365: reward = -362.18, steps = 200\n",
      "02:59:59 [DEBUG] train episode 366: reward = -262.13, steps = 200\n",
      "03:00:35 [DEBUG] train episode 367: reward = -145.10, steps = 200\n",
      "03:01:11 [DEBUG] train episode 368: reward = -268.55, steps = 200\n",
      "03:01:47 [DEBUG] train episode 369: reward = -250.44, steps = 200\n",
      "03:02:23 [DEBUG] train episode 370: reward = -258.48, steps = 200\n",
      "03:02:59 [DEBUG] train episode 371: reward = -139.40, steps = 200\n",
      "03:03:35 [DEBUG] train episode 372: reward = -361.79, steps = 200\n",
      "03:04:11 [DEBUG] train episode 373: reward = -152.58, steps = 200\n",
      "03:04:47 [DEBUG] train episode 374: reward = -256.90, steps = 200\n",
      "03:05:23 [DEBUG] train episode 375: reward = -252.20, steps = 200\n",
      "03:05:59 [DEBUG] train episode 376: reward = -139.85, steps = 200\n",
      "03:06:35 [DEBUG] train episode 377: reward = -469.02, steps = 200\n",
      "03:07:11 [DEBUG] train episode 378: reward = -135.77, steps = 200\n",
      "03:07:48 [DEBUG] train episode 379: reward = -246.54, steps = 200\n",
      "03:08:23 [DEBUG] train episode 380: reward = -253.41, steps = 200\n",
      "03:09:00 [DEBUG] train episode 381: reward = -395.97, steps = 200\n",
      "03:09:36 [DEBUG] train episode 382: reward = -263.93, steps = 200\n",
      "03:10:12 [DEBUG] train episode 383: reward = -391.88, steps = 200\n",
      "03:10:48 [DEBUG] train episode 384: reward = -389.35, steps = 200\n",
      "03:11:24 [DEBUG] train episode 385: reward = -369.63, steps = 200\n",
      "03:12:00 [DEBUG] train episode 386: reward = -257.44, steps = 200\n",
      "03:12:36 [DEBUG] train episode 387: reward = -413.40, steps = 200\n",
      "03:13:12 [DEBUG] train episode 388: reward = -379.60, steps = 200\n",
      "03:13:48 [DEBUG] train episode 389: reward = -265.46, steps = 200\n",
      "03:14:23 [DEBUG] train episode 390: reward = -369.10, steps = 200\n",
      "03:14:59 [DEBUG] train episode 391: reward = -5.30, steps = 200\n",
      "03:15:35 [DEBUG] train episode 392: reward = -256.47, steps = 200\n",
      "03:16:11 [DEBUG] train episode 393: reward = -241.87, steps = 200\n",
      "03:16:46 [DEBUG] train episode 394: reward = -384.62, steps = 200\n",
      "03:17:23 [DEBUG] train episode 395: reward = -129.81, steps = 200\n",
      "03:17:58 [DEBUG] train episode 396: reward = -248.67, steps = 200\n",
      "03:18:33 [DEBUG] train episode 397: reward = -244.89, steps = 200\n",
      "03:19:09 [DEBUG] train episode 398: reward = -124.78, steps = 200\n",
      "03:19:44 [DEBUG] train episode 399: reward = -133.50, steps = 200\n",
      "03:20:19 [DEBUG] train episode 400: reward = -135.10, steps = 200\n",
      "03:20:53 [DEBUG] train episode 401: reward = -142.30, steps = 200\n",
      "03:21:28 [DEBUG] train episode 402: reward = -133.58, steps = 200\n",
      "03:22:02 [DEBUG] train episode 403: reward = -9.07, steps = 200\n",
      "03:22:37 [DEBUG] train episode 404: reward = -233.33, steps = 200\n",
      "03:23:11 [DEBUG] train episode 405: reward = -130.43, steps = 200\n",
      "03:23:47 [DEBUG] train episode 406: reward = -248.22, steps = 200\n",
      "03:24:21 [DEBUG] train episode 407: reward = -8.88, steps = 200\n",
      "03:24:55 [DEBUG] train episode 408: reward = -125.48, steps = 200\n",
      "03:25:30 [DEBUG] train episode 409: reward = -122.92, steps = 200\n",
      "03:26:04 [DEBUG] train episode 410: reward = -135.57, steps = 200\n",
      "03:26:39 [DEBUG] train episode 411: reward = -126.39, steps = 200\n",
      "03:27:16 [DEBUG] train episode 412: reward = -131.60, steps = 200\n",
      "03:27:51 [DEBUG] train episode 413: reward = -259.66, steps = 200\n",
      "03:28:26 [DEBUG] train episode 414: reward = -133.98, steps = 200\n",
      "03:29:00 [DEBUG] train episode 415: reward = -146.23, steps = 200\n",
      "03:29:35 [DEBUG] train episode 416: reward = -274.51, steps = 200\n",
      "03:30:10 [DEBUG] train episode 417: reward = -393.90, steps = 200\n",
      "03:30:45 [DEBUG] train episode 418: reward = -143.77, steps = 200\n",
      "03:31:19 [DEBUG] train episode 419: reward = -254.26, steps = 200\n",
      "03:31:54 [DEBUG] train episode 420: reward = -147.34, steps = 200\n",
      "03:32:29 [DEBUG] train episode 421: reward = -386.88, steps = 200\n",
      "03:33:02 [DEBUG] train episode 422: reward = -393.22, steps = 200\n",
      "03:33:37 [DEBUG] train episode 423: reward = -375.19, steps = 200\n",
      "03:34:11 [DEBUG] train episode 424: reward = -509.10, steps = 200\n",
      "03:34:44 [DEBUG] train episode 425: reward = -387.57, steps = 200\n",
      "03:35:19 [DEBUG] train episode 426: reward = -511.59, steps = 200\n",
      "03:35:53 [DEBUG] train episode 427: reward = -388.49, steps = 200\n",
      "03:36:28 [DEBUG] train episode 428: reward = -15.63, steps = 200\n",
      "03:37:02 [DEBUG] train episode 429: reward = -513.50, steps = 200\n",
      "03:37:37 [DEBUG] train episode 430: reward = -389.41, steps = 200\n",
      "03:38:12 [DEBUG] train episode 431: reward = -618.31, steps = 200\n",
      "03:38:46 [DEBUG] train episode 432: reward = -391.02, steps = 200\n",
      "03:39:21 [DEBUG] train episode 433: reward = -401.49, steps = 200\n",
      "03:39:55 [DEBUG] train episode 434: reward = -150.33, steps = 200\n",
      "03:40:30 [DEBUG] train episode 435: reward = -268.64, steps = 200\n",
      "03:41:05 [DEBUG] train episode 436: reward = -386.93, steps = 200\n",
      "03:41:39 [DEBUG] train episode 437: reward = -274.03, steps = 200\n",
      "03:42:14 [DEBUG] train episode 438: reward = -127.68, steps = 200\n",
      "03:42:48 [DEBUG] train episode 439: reward = -237.95, steps = 200\n",
      "03:43:23 [DEBUG] train episode 440: reward = -132.51, steps = 200\n",
      "03:43:57 [DEBUG] train episode 441: reward = -143.46, steps = 200\n",
      "03:44:31 [DEBUG] train episode 442: reward = -239.78, steps = 200\n",
      "03:45:05 [DEBUG] train episode 443: reward = -139.46, steps = 200\n",
      "03:45:39 [DEBUG] train episode 444: reward = -247.07, steps = 200\n",
      "03:46:14 [DEBUG] train episode 445: reward = -131.12, steps = 200\n",
      "03:46:48 [DEBUG] train episode 446: reward = -251.54, steps = 200\n",
      "03:47:23 [DEBUG] train episode 447: reward = -135.20, steps = 200\n",
      "03:47:57 [DEBUG] train episode 448: reward = -237.85, steps = 200\n",
      "03:48:31 [DEBUG] train episode 449: reward = -132.34, steps = 200\n",
      "03:49:05 [DEBUG] train episode 450: reward = -261.28, steps = 200\n",
      "03:49:39 [DEBUG] train episode 451: reward = -142.68, steps = 200\n",
      "03:50:14 [DEBUG] train episode 452: reward = -141.20, steps = 200\n",
      "03:50:48 [DEBUG] train episode 453: reward = -22.50, steps = 200\n",
      "03:51:22 [DEBUG] train episode 454: reward = -143.55, steps = 200\n",
      "03:51:57 [DEBUG] train episode 455: reward = -386.95, steps = 200\n",
      "03:52:32 [DEBUG] train episode 456: reward = -377.70, steps = 200\n",
      "03:53:07 [DEBUG] train episode 457: reward = -367.26, steps = 200\n",
      "03:53:41 [DEBUG] train episode 458: reward = -406.48, steps = 200\n",
      "03:54:16 [DEBUG] train episode 459: reward = -270.94, steps = 200\n",
      "03:54:51 [DEBUG] train episode 460: reward = -396.44, steps = 200\n",
      "03:55:25 [DEBUG] train episode 461: reward = -392.85, steps = 200\n",
      "03:56:00 [DEBUG] train episode 462: reward = -260.69, steps = 200\n",
      "03:56:35 [DEBUG] train episode 463: reward = -270.38, steps = 200\n",
      "03:57:10 [DEBUG] train episode 464: reward = -521.54, steps = 200\n",
      "03:57:44 [DEBUG] train episode 465: reward = -150.34, steps = 200\n",
      "03:58:19 [DEBUG] train episode 466: reward = -398.00, steps = 200\n",
      "03:58:53 [DEBUG] train episode 467: reward = -398.03, steps = 200\n",
      "03:59:28 [DEBUG] train episode 468: reward = -561.75, steps = 200\n",
      "04:00:03 [DEBUG] train episode 469: reward = -520.42, steps = 200\n",
      "04:00:37 [DEBUG] train episode 470: reward = -619.00, steps = 200\n",
      "04:01:12 [DEBUG] train episode 471: reward = -377.04, steps = 200\n",
      "04:01:46 [DEBUG] train episode 472: reward = -387.21, steps = 200\n",
      "04:02:20 [DEBUG] train episode 473: reward = -135.25, steps = 200\n",
      "04:02:54 [DEBUG] train episode 474: reward = -137.30, steps = 200\n",
      "04:03:28 [DEBUG] train episode 475: reward = -26.36, steps = 200\n",
      "04:04:02 [DEBUG] train episode 476: reward = -362.67, steps = 200\n",
      "04:04:35 [DEBUG] train episode 477: reward = -147.22, steps = 200\n",
      "04:05:08 [DEBUG] train episode 478: reward = -265.20, steps = 200\n",
      "04:05:42 [DEBUG] train episode 479: reward = -256.98, steps = 200\n",
      "04:06:16 [DEBUG] train episode 480: reward = -253.82, steps = 200\n",
      "04:06:50 [DEBUG] train episode 481: reward = -142.87, steps = 200\n",
      "04:07:24 [DEBUG] train episode 482: reward = -263.40, steps = 200\n",
      "04:07:58 [DEBUG] train episode 483: reward = -369.22, steps = 200\n",
      "04:08:32 [DEBUG] train episode 484: reward = -376.75, steps = 200\n",
      "04:09:07 [DEBUG] train episode 485: reward = -137.94, steps = 200\n",
      "04:09:45 [DEBUG] train episode 486: reward = -138.91, steps = 200\n",
      "04:10:24 [DEBUG] train episode 487: reward = -248.26, steps = 200\n",
      "04:11:00 [DEBUG] train episode 488: reward = -374.20, steps = 200\n",
      "04:11:33 [DEBUG] train episode 489: reward = -139.60, steps = 200\n",
      "04:12:07 [DEBUG] train episode 490: reward = -131.70, steps = 200\n",
      "04:12:41 [DEBUG] train episode 491: reward = -373.46, steps = 200\n",
      "04:13:14 [DEBUG] train episode 492: reward = -135.96, steps = 200\n",
      "04:13:48 [DEBUG] train episode 493: reward = -134.39, steps = 200\n",
      "04:14:23 [DEBUG] train episode 494: reward = -271.40, steps = 200\n",
      "04:14:56 [DEBUG] train episode 495: reward = -141.92, steps = 200\n",
      "04:15:31 [DEBUG] train episode 496: reward = -138.07, steps = 200\n",
      "04:16:05 [DEBUG] train episode 497: reward = -136.79, steps = 200\n",
      "04:16:39 [DEBUG] train episode 498: reward = -125.25, steps = 200\n",
      "04:17:12 [DEBUG] train episode 499: reward = -139.96, steps = 200\n",
      "04:17:46 [DEBUG] train episode 500: reward = -117.19, steps = 200\n",
      "04:18:20 [DEBUG] train episode 501: reward = -246.38, steps = 200\n",
      "04:18:54 [DEBUG] train episode 502: reward = -237.78, steps = 200\n",
      "04:19:20 [DEBUG] train episode 503: reward = -256.88, steps = 200\n",
      "04:19:46 [DEBUG] train episode 504: reward = -124.74, steps = 200\n",
      "04:20:13 [DEBUG] train episode 505: reward = -124.80, steps = 200\n",
      "04:20:40 [DEBUG] train episode 506: reward = -135.13, steps = 200\n",
      "04:21:06 [DEBUG] train episode 507: reward = -135.72, steps = 200\n",
      "04:21:32 [DEBUG] train episode 508: reward = -383.88, steps = 200\n",
      "04:21:58 [DEBUG] train episode 509: reward = -135.57, steps = 200\n",
      "04:22:24 [DEBUG] train episode 510: reward = -133.27, steps = 200\n",
      "04:22:50 [DEBUG] train episode 511: reward = -130.36, steps = 200\n",
      "04:23:17 [DEBUG] train episode 512: reward = -238.47, steps = 200\n",
      "04:23:43 [DEBUG] train episode 513: reward = -265.58, steps = 200\n",
      "04:24:09 [DEBUG] train episode 514: reward = -154.38, steps = 200\n",
      "04:24:36 [DEBUG] train episode 515: reward = -128.59, steps = 200\n",
      "04:25:02 [DEBUG] train episode 516: reward = -31.77, steps = 200\n",
      "04:25:28 [DEBUG] train episode 517: reward = -142.36, steps = 200\n",
      "04:25:54 [DEBUG] train episode 518: reward = -241.07, steps = 200\n",
      "04:26:20 [DEBUG] train episode 519: reward = -129.69, steps = 200\n",
      "04:26:48 [DEBUG] train episode 520: reward = -129.90, steps = 200\n",
      "04:27:15 [DEBUG] train episode 521: reward = -129.24, steps = 200\n",
      "04:27:41 [DEBUG] train episode 522: reward = -10.95, steps = 200\n",
      "04:28:07 [DEBUG] train episode 523: reward = -252.28, steps = 200\n",
      "04:28:33 [DEBUG] train episode 524: reward = -10.58, steps = 200\n",
      "04:28:59 [DEBUG] train episode 525: reward = -252.32, steps = 200\n",
      "04:29:25 [DEBUG] train episode 526: reward = -249.32, steps = 200\n",
      "04:29:51 [DEBUG] train episode 527: reward = -129.62, steps = 200\n",
      "04:30:17 [DEBUG] train episode 528: reward = -133.24, steps = 200\n",
      "04:30:43 [DEBUG] train episode 529: reward = -246.56, steps = 200\n",
      "04:31:09 [DEBUG] train episode 530: reward = -360.84, steps = 200\n",
      "04:31:34 [DEBUG] train episode 531: reward = -134.28, steps = 200\n",
      "04:31:57 [DEBUG] train episode 532: reward = -243.18, steps = 200\n",
      "04:32:19 [DEBUG] train episode 533: reward = -246.08, steps = 200\n",
      "04:32:41 [DEBUG] train episode 534: reward = -143.80, steps = 200\n",
      "04:33:03 [DEBUG] train episode 535: reward = -348.01, steps = 200\n",
      "04:33:25 [DEBUG] train episode 536: reward = -138.30, steps = 200\n",
      "04:33:47 [DEBUG] train episode 537: reward = -146.33, steps = 200\n",
      "04:34:10 [DEBUG] train episode 538: reward = -152.73, steps = 200\n",
      "04:34:32 [DEBUG] train episode 539: reward = -150.42, steps = 200\n",
      "04:34:54 [DEBUG] train episode 540: reward = -140.84, steps = 200\n",
      "04:35:16 [DEBUG] train episode 541: reward = -250.79, steps = 200\n",
      "04:35:37 [DEBUG] train episode 542: reward = -260.76, steps = 200\n",
      "04:35:58 [DEBUG] train episode 543: reward = -261.84, steps = 200\n",
      "04:36:20 [DEBUG] train episode 544: reward = -137.73, steps = 200\n",
      "04:36:42 [DEBUG] train episode 545: reward = -150.29, steps = 200\n",
      "04:37:01 [DEBUG] train episode 546: reward = -147.02, steps = 200\n",
      "04:37:20 [DEBUG] train episode 547: reward = -144.03, steps = 200\n",
      "04:37:39 [DEBUG] train episode 548: reward = -147.71, steps = 200\n",
      "04:37:58 [DEBUG] train episode 549: reward = -155.13, steps = 200\n",
      "04:38:16 [DEBUG] train episode 550: reward = -143.84, steps = 200\n",
      "04:38:35 [DEBUG] train episode 551: reward = -369.86, steps = 200\n",
      "04:38:54 [DEBUG] train episode 552: reward = -29.50, steps = 200\n",
      "04:39:13 [DEBUG] train episode 553: reward = -264.57, steps = 200\n",
      "04:39:31 [DEBUG] train episode 554: reward = -265.22, steps = 200\n",
      "04:39:50 [DEBUG] train episode 555: reward = -148.16, steps = 200\n",
      "04:40:09 [DEBUG] train episode 556: reward = -157.11, steps = 200\n",
      "04:40:28 [DEBUG] train episode 557: reward = -279.21, steps = 200\n",
      "04:40:47 [DEBUG] train episode 558: reward = -369.89, steps = 200\n",
      "04:41:05 [DEBUG] train episode 559: reward = -30.47, steps = 200\n",
      "04:41:24 [DEBUG] train episode 560: reward = -266.86, steps = 200\n",
      "04:41:42 [DEBUG] train episode 561: reward = -152.73, steps = 200\n",
      "04:42:01 [DEBUG] train episode 562: reward = -272.90, steps = 200\n",
      "04:42:20 [DEBUG] train episode 563: reward = -288.95, steps = 200\n",
      "04:42:39 [DEBUG] train episode 564: reward = -146.60, steps = 200\n",
      "04:42:57 [DEBUG] train episode 565: reward = -155.58, steps = 200\n",
      "04:43:16 [DEBUG] train episode 566: reward = -393.99, steps = 200\n",
      "04:43:35 [DEBUG] train episode 567: reward = -145.00, steps = 200\n",
      "04:43:54 [DEBUG] train episode 568: reward = -271.42, steps = 200\n",
      "04:44:13 [DEBUG] train episode 569: reward = -150.45, steps = 200\n",
      "04:44:32 [DEBUG] train episode 570: reward = -372.80, steps = 200\n",
      "04:44:50 [DEBUG] train episode 571: reward = -381.20, steps = 200\n",
      "04:45:09 [DEBUG] train episode 572: reward = -146.33, steps = 200\n",
      "04:45:28 [DEBUG] train episode 573: reward = -148.41, steps = 200\n",
      "04:45:47 [DEBUG] train episode 574: reward = -268.54, steps = 200\n",
      "04:46:05 [DEBUG] train episode 575: reward = -261.68, steps = 200\n",
      "04:46:24 [DEBUG] train episode 576: reward = -152.88, steps = 200\n",
      "04:46:43 [DEBUG] train episode 577: reward = -246.33, steps = 200\n",
      "04:47:02 [DEBUG] train episode 578: reward = -146.65, steps = 200\n",
      "04:47:21 [DEBUG] train episode 579: reward = -145.24, steps = 200\n",
      "04:47:40 [DEBUG] train episode 580: reward = -247.96, steps = 200\n",
      "04:47:59 [DEBUG] train episode 581: reward = -372.28, steps = 200\n",
      "04:48:18 [DEBUG] train episode 582: reward = -355.20, steps = 200\n",
      "04:48:37 [DEBUG] train episode 583: reward = -256.57, steps = 200\n",
      "04:48:56 [DEBUG] train episode 584: reward = -135.31, steps = 200\n",
      "04:49:14 [DEBUG] train episode 585: reward = -30.75, steps = 200\n",
      "04:49:33 [DEBUG] train episode 586: reward = -148.51, steps = 200\n",
      "04:49:52 [DEBUG] train episode 587: reward = -22.21, steps = 200\n",
      "04:50:08 [DEBUG] train episode 588: reward = -483.79, steps = 200\n",
      "04:50:23 [DEBUG] train episode 589: reward = -27.13, steps = 200\n",
      "04:50:39 [DEBUG] train episode 590: reward = -625.40, steps = 200\n",
      "04:50:54 [DEBUG] train episode 591: reward = -267.43, steps = 200\n",
      "04:51:10 [DEBUG] train episode 592: reward = -148.58, steps = 200\n",
      "04:51:26 [DEBUG] train episode 593: reward = -572.71, steps = 200\n",
      "04:51:41 [DEBUG] train episode 594: reward = -137.14, steps = 200\n",
      "04:51:56 [DEBUG] train episode 595: reward = -139.26, steps = 200\n",
      "04:52:12 [DEBUG] train episode 596: reward = -143.63, steps = 200\n",
      "04:52:28 [DEBUG] train episode 597: reward = -160.92, steps = 200\n",
      "04:52:43 [DEBUG] train episode 598: reward = -259.13, steps = 200\n",
      "04:52:59 [DEBUG] train episode 599: reward = -370.46, steps = 200\n",
      "04:53:14 [DEBUG] train episode 600: reward = -17.95, steps = 200\n",
      "04:53:29 [DEBUG] train episode 601: reward = -131.81, steps = 200\n",
      "04:53:45 [DEBUG] train episode 602: reward = -318.74, steps = 200\n",
      "04:54:00 [DEBUG] train episode 603: reward = -251.85, steps = 200\n",
      "04:54:16 [DEBUG] train episode 604: reward = -276.33, steps = 200\n",
      "04:54:31 [DEBUG] train episode 605: reward = -144.22, steps = 200\n",
      "04:54:46 [DEBUG] train episode 606: reward = -399.04, steps = 200\n",
      "04:55:00 [DEBUG] train episode 607: reward = -148.95, steps = 200\n",
      "04:55:15 [DEBUG] train episode 608: reward = -273.41, steps = 200\n",
      "04:55:29 [DEBUG] train episode 609: reward = -497.73, steps = 200\n",
      "04:55:44 [DEBUG] train episode 610: reward = -511.02, steps = 200\n",
      "04:55:58 [DEBUG] train episode 611: reward = -22.77, steps = 200\n",
      "04:56:13 [DEBUG] train episode 612: reward = -264.08, steps = 200\n",
      "04:56:27 [DEBUG] train episode 613: reward = -256.93, steps = 200\n",
      "04:56:42 [DEBUG] train episode 614: reward = -392.95, steps = 200\n",
      "04:56:56 [DEBUG] train episode 615: reward = -389.96, steps = 200\n",
      "04:57:11 [DEBUG] train episode 616: reward = -258.54, steps = 200\n",
      "04:57:26 [DEBUG] train episode 617: reward = -188.62, steps = 200\n",
      "04:57:41 [DEBUG] train episode 618: reward = -249.07, steps = 200\n",
      "04:57:55 [DEBUG] train episode 619: reward = -253.72, steps = 200\n",
      "04:58:10 [DEBUG] train episode 620: reward = -379.30, steps = 200\n",
      "04:58:25 [DEBUG] train episode 621: reward = -136.72, steps = 200\n",
      "04:58:39 [DEBUG] train episode 622: reward = -271.42, steps = 200\n",
      "04:58:54 [DEBUG] train episode 623: reward = -500.82, steps = 200\n",
      "04:59:09 [DEBUG] train episode 624: reward = -365.35, steps = 200\n",
      "04:59:23 [DEBUG] train episode 625: reward = -379.10, steps = 200\n",
      "04:59:36 [DEBUG] train episode 626: reward = -20.71, steps = 200\n",
      "04:59:45 [DEBUG] train episode 627: reward = -237.67, steps = 200\n",
      "04:59:59 [DEBUG] train episode 628: reward = -152.22, steps = 200\n",
      "05:00:05 [DEBUG] train episode 629: reward = -464.92, steps = 200\n",
      "05:00:16 [DEBUG] train episode 630: reward = -146.03, steps = 200\n",
      "05:00:25 [DEBUG] train episode 631: reward = -142.56, steps = 200\n",
      "05:00:32 [DEBUG] train episode 632: reward = -140.50, steps = 200\n",
      "05:00:45 [DEBUG] train episode 633: reward = -23.94, steps = 200\n",
      "05:01:00 [DEBUG] train episode 634: reward = -19.75, steps = 200\n",
      "05:01:14 [DEBUG] train episode 635: reward = -268.09, steps = 200\n",
      "05:01:29 [DEBUG] train episode 636: reward = -134.34, steps = 200\n",
      "05:01:43 [DEBUG] train episode 637: reward = -134.32, steps = 200\n",
      "05:01:57 [DEBUG] train episode 638: reward = -137.16, steps = 200\n",
      "05:02:12 [DEBUG] train episode 639: reward = -258.70, steps = 200\n",
      "05:02:27 [DEBUG] train episode 640: reward = -139.23, steps = 200\n",
      "05:02:41 [DEBUG] train episode 641: reward = -140.13, steps = 200\n",
      "05:02:55 [DEBUG] train episode 642: reward = -247.09, steps = 200\n",
      "05:03:10 [DEBUG] train episode 643: reward = -267.13, steps = 200\n",
      "05:03:22 [DEBUG] train episode 644: reward = -258.63, steps = 200\n",
      "05:03:31 [DEBUG] train episode 645: reward = -380.09, steps = 200\n",
      "05:03:45 [DEBUG] train episode 646: reward = -260.87, steps = 200\n",
      "05:04:00 [DEBUG] train episode 647: reward = -264.98, steps = 200\n",
      "05:04:14 [DEBUG] train episode 648: reward = -130.97, steps = 200\n",
      "05:04:29 [DEBUG] train episode 649: reward = -253.48, steps = 200\n",
      "05:04:43 [DEBUG] train episode 650: reward = -624.01, steps = 200\n",
      "05:04:57 [DEBUG] train episode 651: reward = -491.16, steps = 200\n",
      "05:05:12 [DEBUG] train episode 652: reward = -568.33, steps = 200\n",
      "05:05:26 [DEBUG] train episode 653: reward = -132.46, steps = 200\n",
      "05:05:40 [DEBUG] train episode 654: reward = -263.41, steps = 200\n",
      "05:05:47 [DEBUG] train episode 655: reward = -262.41, steps = 200\n",
      "05:06:02 [DEBUG] train episode 656: reward = -503.54, steps = 200\n",
      "05:06:16 [DEBUG] train episode 657: reward = -490.96, steps = 200\n",
      "05:06:31 [DEBUG] train episode 658: reward = -385.64, steps = 200\n",
      "05:06:46 [DEBUG] train episode 659: reward = -16.94, steps = 200\n",
      "05:07:00 [DEBUG] train episode 660: reward = -387.14, steps = 200\n",
      "05:07:10 [DEBUG] train episode 661: reward = -391.91, steps = 200\n",
      "05:07:22 [DEBUG] train episode 662: reward = -378.95, steps = 200\n",
      "05:07:37 [DEBUG] train episode 663: reward = -382.88, steps = 200\n",
      "05:07:44 [DEBUG] train episode 664: reward = -269.38, steps = 200\n",
      "05:07:50 [DEBUG] train episode 665: reward = -248.68, steps = 200\n",
      "05:08:02 [DEBUG] train episode 666: reward = -257.76, steps = 200\n",
      "05:08:09 [DEBUG] train episode 667: reward = -343.24, steps = 200\n",
      "05:08:23 [DEBUG] train episode 668: reward = -266.97, steps = 200\n",
      "05:08:38 [DEBUG] train episode 669: reward = -147.28, steps = 200\n",
      "05:08:52 [DEBUG] train episode 670: reward = -373.44, steps = 200\n",
      "05:09:01 [DEBUG] train episode 671: reward = -369.80, steps = 200\n",
      "05:09:12 [DEBUG] train episode 672: reward = -146.29, steps = 200\n",
      "05:09:26 [DEBUG] train episode 673: reward = -277.42, steps = 200\n",
      "05:09:41 [DEBUG] train episode 674: reward = -371.51, steps = 200\n",
      "05:09:50 [DEBUG] train episode 675: reward = -257.71, steps = 200\n",
      "05:09:56 [DEBUG] train episode 676: reward = -695.70, steps = 200\n",
      "05:10:07 [DEBUG] train episode 677: reward = -267.22, steps = 200\n",
      "05:10:15 [DEBUG] train episode 678: reward = -141.26, steps = 200\n",
      "05:10:30 [DEBUG] train episode 679: reward = -330.30, steps = 200\n",
      "05:10:44 [DEBUG] train episode 680: reward = -139.12, steps = 200\n",
      "05:10:59 [DEBUG] train episode 681: reward = -391.35, steps = 200\n",
      "05:11:13 [DEBUG] train episode 682: reward = -376.18, steps = 200\n",
      "05:11:28 [DEBUG] train episode 683: reward = -147.17, steps = 200\n",
      "05:11:42 [DEBUG] train episode 684: reward = -145.56, steps = 200\n",
      "05:11:50 [DEBUG] train episode 685: reward = -149.14, steps = 200\n",
      "05:12:04 [DEBUG] train episode 686: reward = -21.33, steps = 200\n",
      "05:12:18 [DEBUG] train episode 687: reward = -262.64, steps = 200\n",
      "05:12:33 [DEBUG] train episode 688: reward = -499.90, steps = 200\n",
      "05:12:47 [DEBUG] train episode 689: reward = -257.49, steps = 200\n",
      "05:13:02 [DEBUG] train episode 690: reward = -337.39, steps = 200\n",
      "05:13:17 [DEBUG] train episode 691: reward = -379.15, steps = 200\n",
      "05:13:31 [DEBUG] train episode 692: reward = -258.72, steps = 200\n",
      "05:13:46 [DEBUG] train episode 693: reward = -370.76, steps = 200\n",
      "05:13:52 [DEBUG] train episode 694: reward = -494.34, steps = 200\n",
      "05:14:07 [DEBUG] train episode 695: reward = -501.50, steps = 200\n",
      "05:14:21 [DEBUG] train episode 696: reward = -390.46, steps = 200\n",
      "05:14:36 [DEBUG] train episode 697: reward = -27.81, steps = 200\n",
      "05:14:44 [DEBUG] train episode 698: reward = -246.19, steps = 200\n",
      "05:14:58 [DEBUG] train episode 699: reward = -485.25, steps = 200\n",
      "05:15:13 [DEBUG] train episode 700: reward = -606.27, steps = 200\n",
      "05:15:27 [DEBUG] train episode 701: reward = -136.10, steps = 200\n",
      "05:15:37 [DEBUG] train episode 702: reward = -272.39, steps = 200\n",
      "05:15:46 [DEBUG] train episode 703: reward = -264.44, steps = 200\n",
      "05:16:01 [DEBUG] train episode 704: reward = -483.69, steps = 200\n",
      "05:16:11 [DEBUG] train episode 705: reward = -377.25, steps = 200\n",
      "05:16:23 [DEBUG] train episode 706: reward = -587.57, steps = 200\n",
      "05:16:38 [DEBUG] train episode 707: reward = -378.68, steps = 200\n",
      "05:16:52 [DEBUG] train episode 708: reward = -256.20, steps = 200\n",
      "05:17:03 [DEBUG] train episode 709: reward = -140.29, steps = 200\n",
      "05:17:13 [DEBUG] train episode 710: reward = -266.18, steps = 200\n",
      "05:17:28 [DEBUG] train episode 711: reward = -252.68, steps = 200\n",
      "05:17:42 [DEBUG] train episode 712: reward = -362.65, steps = 200\n",
      "05:17:52 [DEBUG] train episode 713: reward = -20.84, steps = 200\n",
      "05:18:06 [DEBUG] train episode 714: reward = -257.77, steps = 200\n",
      "05:18:19 [DEBUG] train episode 715: reward = -253.80, steps = 200\n",
      "05:18:28 [DEBUG] train episode 716: reward = -147.25, steps = 200\n",
      "05:18:42 [DEBUG] train episode 717: reward = -149.85, steps = 200\n",
      "05:18:57 [DEBUG] train episode 718: reward = -21.55, steps = 200\n",
      "05:19:11 [DEBUG] train episode 719: reward = -256.47, steps = 200\n",
      "05:19:26 [DEBUG] train episode 720: reward = -126.24, steps = 200\n",
      "05:19:40 [DEBUG] train episode 721: reward = -137.32, steps = 200\n",
      "05:19:54 [DEBUG] train episode 722: reward = -134.54, steps = 200\n",
      "05:20:09 [DEBUG] train episode 723: reward = -141.33, steps = 200\n",
      "05:20:24 [DEBUG] train episode 724: reward = -258.30, steps = 200\n",
      "05:20:39 [DEBUG] train episode 725: reward = -262.27, steps = 200\n",
      "05:20:54 [DEBUG] train episode 726: reward = -128.37, steps = 200\n",
      "05:21:08 [DEBUG] train episode 727: reward = -265.61, steps = 200\n",
      "05:21:23 [DEBUG] train episode 728: reward = -243.92, steps = 200\n",
      "05:21:37 [DEBUG] train episode 729: reward = -137.17, steps = 200\n",
      "05:21:52 [DEBUG] train episode 730: reward = -487.09, steps = 200\n",
      "05:22:07 [DEBUG] train episode 731: reward = -144.12, steps = 200\n",
      "05:22:22 [DEBUG] train episode 732: reward = -127.64, steps = 200\n",
      "05:22:36 [DEBUG] train episode 733: reward = -265.59, steps = 200\n",
      "05:22:51 [DEBUG] train episode 734: reward = -12.00, steps = 200\n",
      "05:23:05 [DEBUG] train episode 735: reward = -130.40, steps = 200\n",
      "05:23:20 [DEBUG] train episode 736: reward = -129.79, steps = 200\n",
      "05:23:34 [DEBUG] train episode 737: reward = -130.46, steps = 200\n",
      "05:23:49 [DEBUG] train episode 738: reward = -134.46, steps = 200\n",
      "05:24:04 [DEBUG] train episode 739: reward = -132.38, steps = 200\n",
      "05:24:18 [DEBUG] train episode 740: reward = -142.86, steps = 200\n",
      "05:24:33 [DEBUG] train episode 741: reward = -129.09, steps = 200\n",
      "05:24:47 [DEBUG] train episode 742: reward = -248.97, steps = 200\n",
      "05:25:02 [DEBUG] train episode 743: reward = -129.60, steps = 200\n",
      "05:25:12 [DEBUG] train episode 744: reward = -130.07, steps = 200\n",
      "05:25:22 [DEBUG] train episode 745: reward = -128.66, steps = 200\n",
      "05:25:37 [DEBUG] train episode 746: reward = -131.91, steps = 200\n",
      "05:25:51 [DEBUG] train episode 747: reward = -128.17, steps = 200\n",
      "05:26:06 [DEBUG] train episode 748: reward = -131.47, steps = 200\n",
      "05:26:21 [DEBUG] train episode 749: reward = -130.54, steps = 200\n",
      "05:26:35 [DEBUG] train episode 750: reward = -9.30, steps = 200\n",
      "05:26:45 [DEBUG] train episode 751: reward = -140.80, steps = 200\n",
      "05:26:55 [DEBUG] train episode 752: reward = -128.94, steps = 200\n",
      "05:26:55 [INFO] ==== test ====\n",
      "05:26:55 [DEBUG] test episode 0: reward = -9.61, steps = 200\n",
      "05:26:55 [DEBUG] test episode 1: reward = -9.75, steps = 200\n",
      "05:26:55 [DEBUG] test episode 2: reward = -252.07, steps = 200\n",
      "05:26:55 [DEBUG] test episode 3: reward = -135.83, steps = 200\n",
      "05:26:56 [DEBUG] test episode 4: reward = -12.33, steps = 200\n",
      "05:26:56 [DEBUG] test episode 5: reward = -244.85, steps = 200\n",
      "05:26:56 [DEBUG] test episode 6: reward = -369.31, steps = 200\n",
      "05:26:56 [DEBUG] test episode 7: reward = -136.45, steps = 200\n",
      "05:26:56 [DEBUG] test episode 8: reward = -135.23, steps = 200\n",
      "05:26:56 [DEBUG] test episode 9: reward = -9.71, steps = 200\n",
      "05:26:56 [DEBUG] test episode 10: reward = -9.52, steps = 200\n",
      "05:26:56 [DEBUG] test episode 11: reward = -11.22, steps = 200\n",
      "05:26:56 [DEBUG] test episode 12: reward = -131.48, steps = 200\n",
      "05:26:57 [DEBUG] test episode 13: reward = -370.97, steps = 200\n",
      "05:26:57 [DEBUG] test episode 14: reward = -254.68, steps = 200\n",
      "05:26:57 [DEBUG] test episode 15: reward = -134.99, steps = 200\n",
      "05:26:57 [DEBUG] test episode 16: reward = -367.24, steps = 200\n",
      "05:26:57 [DEBUG] test episode 17: reward = -257.25, steps = 200\n",
      "05:26:57 [DEBUG] test episode 18: reward = -251.54, steps = 200\n",
      "05:26:57 [DEBUG] test episode 19: reward = -254.37, steps = 200\n",
      "05:26:57 [DEBUG] test episode 20: reward = -10.54, steps = 200\n",
      "05:26:57 [DEBUG] test episode 21: reward = -365.44, steps = 200\n",
      "05:26:57 [DEBUG] test episode 22: reward = -254.77, steps = 200\n",
      "05:26:58 [DEBUG] test episode 23: reward = -137.39, steps = 200\n",
      "05:26:58 [DEBUG] test episode 24: reward = -135.34, steps = 200\n",
      "05:26:58 [DEBUG] test episode 25: reward = -436.01, steps = 200\n",
      "05:26:58 [DEBUG] test episode 26: reward = -130.01, steps = 200\n",
      "05:26:58 [DEBUG] test episode 27: reward = -136.92, steps = 200\n",
      "05:26:58 [DEBUG] test episode 28: reward = -252.89, steps = 200\n",
      "05:26:58 [DEBUG] test episode 29: reward = -131.79, steps = 200\n",
      "05:26:58 [DEBUG] test episode 30: reward = -134.19, steps = 200\n",
      "05:26:58 [DEBUG] test episode 31: reward = -368.75, steps = 200\n",
      "05:26:59 [DEBUG] test episode 32: reward = -138.49, steps = 200\n",
      "05:26:59 [DEBUG] test episode 33: reward = -319.54, steps = 200\n",
      "05:26:59 [DEBUG] test episode 34: reward = -256.56, steps = 200\n",
      "05:26:59 [DEBUG] test episode 35: reward = -339.64, steps = 200\n",
      "05:26:59 [DEBUG] test episode 36: reward = -244.61, steps = 200\n",
      "05:26:59 [DEBUG] test episode 37: reward = -10.47, steps = 200\n",
      "05:26:59 [DEBUG] test episode 38: reward = -137.76, steps = 200\n",
      "05:26:59 [DEBUG] test episode 39: reward = -242.82, steps = 200\n",
      "05:26:59 [DEBUG] test episode 40: reward = -361.32, steps = 200\n",
      "05:27:00 [DEBUG] test episode 41: reward = -248.25, steps = 200\n",
      "05:27:00 [DEBUG] test episode 42: reward = -129.70, steps = 200\n",
      "05:27:00 [DEBUG] test episode 43: reward = -253.74, steps = 200\n",
      "05:27:00 [DEBUG] test episode 44: reward = -253.95, steps = 200\n",
      "05:27:00 [DEBUG] test episode 45: reward = -250.36, steps = 200\n",
      "05:27:00 [DEBUG] test episode 46: reward = -130.20, steps = 200\n",
      "05:27:00 [DEBUG] test episode 47: reward = -292.78, steps = 200\n",
      "05:27:00 [DEBUG] test episode 48: reward = -128.97, steps = 200\n",
      "05:27:00 [DEBUG] test episode 49: reward = -248.38, steps = 200\n",
      "05:27:00 [DEBUG] test episode 50: reward = -9.78, steps = 200\n",
      "05:27:01 [DEBUG] test episode 51: reward = -360.73, steps = 200\n",
      "05:27:01 [DEBUG] test episode 52: reward = -130.29, steps = 200\n",
      "05:27:01 [DEBUG] test episode 53: reward = -258.35, steps = 200\n",
      "05:27:01 [DEBUG] test episode 54: reward = -251.22, steps = 200\n",
      "05:27:01 [DEBUG] test episode 55: reward = -136.61, steps = 200\n",
      "05:27:01 [DEBUG] test episode 56: reward = -131.08, steps = 200\n",
      "05:27:01 [DEBUG] test episode 57: reward = -481.58, steps = 200\n",
      "05:27:01 [DEBUG] test episode 58: reward = -252.20, steps = 200\n",
      "05:27:01 [DEBUG] test episode 59: reward = -376.63, steps = 200\n",
      "05:27:02 [DEBUG] test episode 60: reward = -245.14, steps = 200\n",
      "05:27:02 [DEBUG] test episode 61: reward = -132.08, steps = 200\n",
      "05:27:02 [DEBUG] test episode 62: reward = -252.34, steps = 200\n",
      "05:27:02 [DEBUG] test episode 63: reward = -369.04, steps = 200\n",
      "05:27:02 [DEBUG] test episode 64: reward = -132.32, steps = 200\n",
      "05:27:02 [DEBUG] test episode 65: reward = -132.45, steps = 200\n",
      "05:27:02 [DEBUG] test episode 66: reward = -131.34, steps = 200\n",
      "05:27:02 [DEBUG] test episode 67: reward = -347.01, steps = 200\n",
      "05:27:02 [DEBUG] test episode 68: reward = -10.80, steps = 200\n",
      "05:27:03 [DEBUG] test episode 69: reward = -135.18, steps = 200\n",
      "05:27:03 [DEBUG] test episode 70: reward = -130.36, steps = 200\n",
      "05:27:03 [DEBUG] test episode 71: reward = -139.63, steps = 200\n",
      "05:27:03 [DEBUG] test episode 72: reward = -132.52, steps = 200\n",
      "05:27:03 [DEBUG] test episode 73: reward = -126.73, steps = 200\n",
      "05:27:03 [DEBUG] test episode 74: reward = -132.41, steps = 200\n",
      "05:27:03 [DEBUG] test episode 75: reward = -10.50, steps = 200\n",
      "05:27:03 [DEBUG] test episode 76: reward = -357.46, steps = 200\n",
      "05:27:03 [DEBUG] test episode 77: reward = -506.79, steps = 200\n",
      "05:27:03 [DEBUG] test episode 78: reward = -136.58, steps = 200\n",
      "05:27:04 [DEBUG] test episode 79: reward = -363.59, steps = 200\n",
      "05:27:04 [DEBUG] test episode 80: reward = -254.00, steps = 200\n",
      "05:27:04 [DEBUG] test episode 81: reward = -247.59, steps = 200\n",
      "05:27:04 [DEBUG] test episode 82: reward = -138.31, steps = 200\n",
      "05:27:04 [DEBUG] test episode 83: reward = -11.14, steps = 200\n",
      "05:27:04 [DEBUG] test episode 84: reward = -254.55, steps = 200\n",
      "05:27:04 [DEBUG] test episode 85: reward = -253.99, steps = 200\n",
      "05:27:04 [DEBUG] test episode 86: reward = -339.10, steps = 200\n",
      "05:27:04 [DEBUG] test episode 87: reward = -365.86, steps = 200\n",
      "05:27:05 [DEBUG] test episode 88: reward = -370.71, steps = 200\n",
      "05:27:05 [DEBUG] test episode 89: reward = -256.71, steps = 200\n",
      "05:27:05 [DEBUG] test episode 90: reward = -367.21, steps = 200\n",
      "05:27:05 [DEBUG] test episode 91: reward = -131.47, steps = 200\n",
      "05:27:05 [DEBUG] test episode 92: reward = -252.01, steps = 200\n",
      "05:27:05 [DEBUG] test episode 93: reward = -11.49, steps = 200\n",
      "05:27:05 [DEBUG] test episode 94: reward = -252.88, steps = 200\n",
      "05:27:05 [DEBUG] test episode 95: reward = -130.58, steps = 200\n",
      "05:27:05 [DEBUG] test episode 96: reward = -255.21, steps = 200\n",
      "05:27:05 [DEBUG] test episode 97: reward = -255.73, steps = 200\n",
      "05:27:06 [DEBUG] test episode 98: reward = -9.77, steps = 200\n",
      "05:27:06 [DEBUG] test episode 99: reward = -370.92, steps = 200\n",
      "05:27:06 [INFO] average episode reward = -206.82 ± 120.74\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABGl0lEQVR4nO2deZwU1dX3f6e7Z2Eb1mEdYEBAQJBtRFzABRRUInFBMa6vMURjYoyJCcbo45poTPRxSVwSH6NR424wihtRURTFQXYBHRZhWIdtZlhm6en7/tF1u6uqby3dXT1TPX2+n09D9a2qW2eqq+6555x7zyUhBBiGYRhGEmhpARiGYRh/wYqBYRiGMcCKgWEYhjHAioFhGIYxwIqBYRiGMRBqaQHSpVu3bqK0tLSlxWAYhskqlixZslsIUazal/WKobS0FOXl5S0tBsMwTFZBRN9Z7WNXEsMwDGOAFQPDMAxjgBUDwzAMY4AVA8MwDGOAFQPDMAxjwHeKgYimEdE6IqogojktLQ/DMEyu4SvFQERBAH8BcAaA4QAuIqLhLSsVwzBMbuErxQBgPIAKIcQGIUQDgBcAzGhhmRiXrKjcj817DuGl8i1oikTTuX+35yAWrd/TwpL5i027D4LT3TN+xm+KoQ+ALbrvlVqZASKaTUTlRFReVVXVbMLlMqc/sADn/vVT22POfuRTTLrvQ/z6lRV47ovo3JmT7vsIF/3t85Sv2xQRKJ3zFh7+77cAgKueLsfP/rXU9py5y7ZiyXf7DGXhpgge/Wg9Djc0pSyLF3y+YQ9O/tNHeHlJZazsm521OOevn+KuN7/GXW9+bXt+JOI/hRJuiuC3r6/ErXNX+VK+bKW2rhGvfVWp3HfbG6vx5optGbu23xQDKcoSnjQhxBNCiDIhRFlxsXJGN+Mx3+w8gK8273d9fM3hRk+uWx+ONuR/+agCADB/zU78Z7n9C/HzF5bhvEc/M5S9tnQr7n1nLR7UFExLUbHrAABg2Zb9sbI/zFuDpZv34+8LN+LvCzdanrt08z4M/O08fFax2/E6q7ZWo7Epkra8bnhr5XY8/8VmPLPoO5SbFLLXVOw6gKraekPZC4s345cvLTeUrd1Ro1RSryypxP5DDRBC4B+fbnR1L699/itc9fSXruRbWVmNil21ro5d+O1uVOyqxYrK/diy91DC/t/9exVueGk5VlTuT9j3j8824afP23eQ0sFviqESQF/d9xIAmVOLTMYIBrx5tBrD0Zc7lEJ91zy7JLZdWxcGANQ1qi2GusYmnP3IQizdnNmGjbSuj96VFHbZy160IeqSW/CtvZW8afdBTH94Ie5+a43lMTuq6zDuzvddN2Iqqg83Yuv+wziks8IiLl1kr31ViV01dRBC4J1VO2KuRzu27D2EKfcvwDF3z8dXut9pzmsr8aquZ72icj+m/e8neHTBesP5G6oO4FcvL8d1LyzDyq3VuO0/X+OWuascr/vWiu2Yv2aXq7/re48sxJT7P4593159GDe9tlKppC958gtMuf9jnP3Ip5j4xw8T9u+qiSpA+ew2J35TDF8CGExEA4goH8AsAG+0sExMCuQFVcZf8kiLIZBCdW+v2hHbbopEbOVat6MWKyqr8T9vrE7+QklAmlGsbz/dNqbmOqzYczDaoCxX9DQl81Zux56DDXj2881JXVvPmQ9+ghPu+cBVo66n+lAjbnhpOS5/6ku8s2oHrn52CR7/eL3jefrG8/oXllket21/HQBguc4qA4AGrXHeWV2HPQcaACDjrsXfvrYS/1q8GQu/dbZMzIS0ZzXZ++sFvkqiJ4QIE9FPAbwLIAjg/4QQmX1TmYyQF/Smz1Efjr7MoTTrk71yK0smqGmexqbMvoRxiyFe5vbF9zJeLXuw6SjwrfsPA4jGGCRuZGzSDtpRfRi7D0SV2NZ9h5O6NtmILTsR5tsqFWpEiFiHI93nyi3JKn8g/kyGI0ZrI9wMLkJfKQYAEELMAzCvpeVg0iPkmcUQfQkCdi2BC8JN0iWlrkcqsky/dPLqQhc6iyR5yTRvBYC4YvCiYXTrCpNI8SMCsT/GC50nhAARxZ4V88ivuMIQug6HN8+pFXFZkj9Xuk/Dps5KQzMoBr+5kphWQp5HMYZYzy4VX5KOcJN9QyB7Z5k225UWQ9KuJHvMDYkKaRl5Ydnp75kbpSWPFkLEFWWSt111mVgnQvuTzL100jXS9Y3a85Dmc+UE6ZRRsoQsnsmGMCsGJkvxqicmX4Jgmi9wY8SpIYzuT7b3myyxGIOuzO01ZQ/YqfGVPUq7w6R7Is+DhvGwRUDfCn1DF7cE07/vUjFIBZDgSpJKGfEOh1eDJKygNCyioPYONZr+kHpWDEy24n2MIb0GrCkWY1DXI63zjLuSFBaDeVilldXittPppkcZsxhC6f9OB+uTGzUjFZzOk5SCOy3xd5SNfdxVZbxh8qvBlZRpiyF27XQsBuPNkdZOJmHFwHiCuXHz6oWLWQwZjjHIxjjjFkOsBxm/TrKuAqdRSW56lPIaXijwA/XJDVeVrjMh1DGXVJENppVfP6K7br1HlqgeVacinRhDLPicEGPI/CRNVgyMJyTrJ3eL1QucbA9Muk6sFINsNDLvStKwGZUke75m3Epmdb6esMPw3WTQWwxuev7y700nLYg6xiCHNicqXyD+G+stBi8NBlVQOB5jSL4+Gacz11vHFgOTLZgbN6/aV6sYQ7IjM6TrJGjRQ5byZ3q2sCoYaVaqjhaDQ2PmxtUQt6C8dSWZh1aqkIcIpNejNlPXaGzszaLEFVJciXjZD1D9blZKyg0yxmD+PTnGwPieZVv2oz7clNDTTteCqNhVi501dbrx5sbWsK4hklSPM+antThHNtTNNipJV2a+plWP0O2faw7CqnCrWMNNEdz+n9XYuPug5TEHG3QWgwsh9T13uOxRu8nBlBh8VscYhBCxxtbL31up0NOwGKR1a1YEzTEqyXfzGBhviUQEmoRIy5cs8/uYkbNWAWBYryLDvuv+tRTX2SS7e+Lj9RjcowNCAcK4/p3RNj+E7dWHcbA+jIJQMJZW4J5zRwKIxhhq6uL5l0bd8R4AYHTfTvjLxWOxams1ph7VM+E6jU0R5AUDWLW1BgBQUxfG1v2H0bOoEI1NERTmBQHoXElNApv3HMLlTy3Gi7MnoHtRofMNSgIZH6g53Ih3V+/Akws3xiZ5SSbd9yHm3zAJg7p3sK0rEhF49ovvMHNcXxTmBXCgPowOhXlJBZ9VDbm8ZwAw/eGFWLujFs8s+g79u7TF7TOOwth+nQ2N1ecb9sa2F2/chxMHFSPfJqidSowhoZes03mhACEciU9asxoiKpVARBdjULkOqw81omPbPFt5XMkIWM6pUHVq1u6oQdu8EHbW1mFM30745+fRRJSffFuFC4/pi8Ub96L6cAOe/uy72Dkbdx/EgG7tkpbVCVYMrZh/LtqEW+ZGJ45vuueshP27D9Tjuc8349pTjrCd6DTl/gXK8qt1uYjWbK9xLdc3O2vx+3lrDWXr7pqGG15cHssHJJnz2koAwPLKahx923sJdS3bsh+T//xRrJfdRmvoJYNvfht/njkKX2vy3ffuOtz37jpcdlx/vLB4C1bdPhX5oUB8VFIkgqcXbcLG3QfxxvJtuGriQMu/40B9GPsONqBvl7aG8m37D6N3pzZYUbkfizfuNdQhG60P11Xhw3XWOY+m3P8x7jv/aMwsi6cOk42nbBPfX7MTt85djVvnrsaFZX3xYvkWRU1qZCJCfeP54dpdiAiBHz5djg4FIVxyXH+s3RHNpdQUEdiw+yAufXIxAKBPpzbKeh9bsB6PLViPWcf0xU1nDsOWvYcwvFcRhFb/5GHdY71/vSvJydOiynH1/tc7MaBbO4SCmmLQnoH46KP4sS+Vb0EPTckLCNRr9a3ZXoPr/rUUsycNxPBeRfh6ew2mP7wQN50xFH94ey2evnK8vWAaG3cfVFof8rf6tGI3OrfNx6QhxYhEhNJim/a/n8S2TxjUNfZ3fLZ+D8be+b7yul9s2MOKgUkOqRRU7Kqtw/i7/wsAqNx3CJOH9cC0EYk9bq/5elsNznzok4TyI3/3Tsp16l0vqjH1v3x5eULZM4uiva7TH1iAAd3aoUNhtIcYEe5dNhc+vgirt9XgjhlH4ZJj++PNlduxams1nvh4A3580kA8vmADAOC8sSXo3C4fgHPdA4vbYUNV1G1z4ysr8NE3VbjyhFKM698lfq7WmOrz/KiUgpWbRG9RLN28Hztr1mJ3bYOhjtr6MB79yDp/kUyHYcULX27BC1/G6+vVsRDbq6M5jPppirQhHIm5oSJC4N3VO7D7QD2OHdAVew7Uo3O7fHRsk4eXvtyCU4d1N9S/u7YeP3qm3FD2+YY9qNh1AIN6tAcQ75Wv3laNX7+yAt3aFwAAdtbUY96q7bHz3li+DW8s34bfnTUMRW2iz8EjH0Sz+T73+Xcws6umDjtq6rDvUCMmDOwSe3aHm6xmIB7veKm8Ei+VV2LTPWfh9v+sxtOLEuvV82lFvIN0QVkJXiqPJwm88/sj8MD732DvwYaE++IVrBhylC174y/2y0sq8fKSSqVV4RUyXcGu2rqMXSMVNu05hE17jCmPYz1zhyjv6m1RK+TWuavRvUOhwXUmlQJgDGirGutzx/bBa19tBQCccmR3DO9VhzdXRBuut1Zsx1srtmPNHdNixxOAsx76JHZ9K/SB4NXbqnHWQwvx9s8norRrvIf5+tKttnXcMn047nRYI8INUikAwGZdiulbtc7Lv5dtw7+XWSdSPmhKdlejyDj6+McbDN/lrZaKUO+yq2uMID8YMPTct1fXoaOmGGRMSzUYYfzv/xvb/uhXJ8e2v1ZYzapnyEkpmPnj+aPwzqodqKkLY9KQYlw6oT/OGtkL3+ysRfcO3ro6JRx8zhHMPs1Upuind/3o/3a+Z78gZU1mKOOG3eo4DGCcuaq677+ZNjS23RQRynsUjkQM3hYnpRCtK779jpZp9r3VO12NHJJ071CAAg9/sytPGJDSedWHG5I+Rz7zVnMVRvftZDo+3pDLGdFOw5frFEODOxTG+9tejYa99pRBhu9d2uVjwsCuHtWeiP/fUsYTzFlDmzuVr7yal41MpoilnkjinEqb7KB6141KMejvSTgSUd4jfVvudq6ffsasfmJdMrOM80OBtJW53sXy/04oTamO6hQWforEFLz93BWJXmGGAtYWgx6ntN1OVqdbmvu98f9byniCOdjV3IpBvoQFoaDDkS1P3JXv/qXebuNz1084U912feA/+rskXrdJJBH80ND3dvUZTZOxGPJDgbQbpYK8+PmpKpmaw8kvViOfOauf0awY6hqbYvfJbVLFA6Z0IHlBSjqpoBsK8pr3veEYQ47QEI4ABfHvLaUY0k2f3Rw4NSgqDtn0HPXBcdV918/GbmwSULUB+sbcKSWG6lrxXHUiqd++IBhAfpppMwp1nYFUlUwqFoMQ0dFXSy2WpDUnodUPN3W7PscBU6yjIBQ0dMKScUcW5gUs57A0t8XAiiFHMI9tz1QKCyv0Ccz8jhQxGRVml2G0XrdPNX5drxiaIkKZpqIpInSWjDuZ9Dl29Fldk/nt9a6kYIBS6lB4YjHUpeJKEviZzVwac3K6usam+ByLWLZde+vKvOxmQShgeBbcKnEA6FCYh7rGeuW+5o7NsSspRzArBjc5+70kC/RBjFRcSXa+Zn1PVNWw6oOjVsHOcJNISmERmdNbR/+PCJHUb69XDOY5Im7RWwypWh81KVoMdpjb/LrGSCymIPc53atakyspOidGxDoAyWQc0QetzTS3C5YVQ45gjjGkO63eTYoCw/H6NAg+waqRcrvugR47i0E/OUt12/QKyNyLjZeLpPLtFIQCBssgEIjnJErKlRQKxhVDfmqNk7QYggFCKBhIye+eiivJyTIyP4v14abYeyH3OQWfa02WjHT5yFucTOdCzqVRken04GZYMeQIZkXgJgOnHck28PJoH+kFS7+t7CUm4wZwazE43TerRjucZEAzLxBQ1pWKK0n2VtumqBikxSDvdypNnJUus0ub7XSvzdZZXWMk9p64TcOuijFEz1Nnb7VTykU2FoO8ac2lHjKmGIjoPiJaS0QriOh1IuqklZcS0WEiWqZ9HtOdM46IVhJRBRE9RF6N9WI8txiSjVH40WKwGulRl0JKZtsYg4MrSU/UDZFYHhHqciuCQTKsD6BfICgZiyE/FA8+p+pKkhaDtDy8fK3b2snk6EpKHJUUtxiiZU6uJPOoJPm3ynts7lzYvXd2riT5tzTX25NJi+F9ACOEEEcD+AbATbp964UQo7XP1bryRwHMBjBY+0wD4wnmBzLVBcX3aLNHk23fhS7Vsl+wshik68e7UUn2riQgmgoDsI8xJEPIFCiOBZ+TjTEEA7H8Vd/srE1KBom8z/L/dLwi5sC8nXvLKT21Kt25fC8aTf8DUUViHjygCj4D8d/R3BGyUwxt8vwzFihjikEI8Z4QQt61zwGU2B1PRL0AFAkhFono3X8GwPczJV+uYfaVOuXsP9zQhC9MCe0AYNxd81FVW2/bEKqoOlCH0jlv4cO1u5I6L5PoR8voiaVv1hrTA/Vh/PyFpdh3MPnZtwCwSZey2txQSIVw/wWjAQC9LZLTGYeeGhvH135yfMLxQS3jqEQ2xgLJWwwyx5DqtEd+MMYx0ZzMYCsztqZjMZiDsHaKwWkkk/k+1NaHYw23VOb692b1thocfbsxkaP5GlK+JovstXYdMjtXndzXrX2+5TFe0lwxhisBvK37PoCIlhLRAiKaqJX1AVCpO6ZSK0uAiGYTUTkRlVdVWWeoZOK4sRj0pvWc11bgwic+x5a9hxKOO+bu+fjB3z43lA3q3t72+pt2R+t5WEtO5kSfTm0w99oTYt+H9GiPy4/r7+pct3RrV6Asj/Xwtfbr+S++w9xl2/CXDytQue8QdtUkl+/ppfItWPLdPpTOeSthTP2rV0cb9dF9O+GxS8bh1unDlQ2nOSWGnrH9OuPzmybj+CO6xnrUoUAg1ihtqDqARxdEE+IJIZJyAxaEAvjvL08CoO7pTz+6N04aUoxzxkRfVdVzcERxtEzODk9WLRxT2hlAVEmZz7Vzb5l782bMiqGqtj6Wb0l2DvYdijf8ry/dmlDnJ9/uNnwv1DobY+58H9Mf/sSw7CmQ6HrS09km1ff4AV1w9zkjcPvZR1ke4yVpKQYimk9EqxSfGbpjbgYQBvCcVrQdQD8hxBgANwB4noiKoH5elE+wEOIJIUSZEKKsuLg4nT8hZ9Cb1UIIbK9OnKmr72HKNNp7tF5y+wKjmSvTMQPAU1ccgxdmT8ADF44yHLP8f07Hb8+M5gFyu7auPn2Cfo2H935xEgZqDcylE/rjkR+McVWfZM4ZQw3f+3Zpg6I20b+prH9nw76tpgZMnzfnxHs/NCRRc0NNXRjnPfoZAGD+mp0GGWTWVQCYNqInCvOC+NXpQzBxcDdDHVaxB0nPjoV4/kcT8Oo1x6N9QQiThhQjHBH4ZmctLnj8c+zXGrhojCGJmc/BADq2ycPca0/AO9dPsjzugQtHY9M9Z8V+bz2nH9UDQ3t2wHljo06DZAwGIqBjm+g9Urn+2hWk7n6RMY9bpw93lcNpp4sEkPr5Bqu21mBF5X7D/t+8usLwfeLgblh1+1S89pPjce5Ya6cKEeHiY/vbjlzykrQUgxBiihBihOIzFwCI6HIA0wFcrLmHIISoF0Ls0baXAFgPYAiiFoL+zpQAsE63yCSF3kJ4ZUklnv18c8IxxnHv0bdX9p4LbXpm7QtD6Na+IJZOWdKxTR7a5EdfXLdLZuplMCuTuDtEJD2DWvY6JfNvOAmy6TevuSBTSktJpFvbTeB8ShJpkK3a567tC/DgLKPiC+uGq9qtXHd0SSesun0qehYVQgjg9Ac+NmQVjYjEeMWfZo4yVxNDDnMd1beT4fd99/pJ+OTXpyQcrxpv3zY/hHeun4Q/XxC9jvzt9MkDVbQvCOH9X5wUc/mpnsFUR0oB0Q7NdZMH4/+dUJrQcVBhl/ZEYv77zVbJrpo6tNPJ/IPx/dC+IISx/TqjtFs7bPzDmfjxSfHn8eMbE+9xc5DJUUnTAPwGwNlCiEO68mIiCmrbAxENMm8QQmwHUEtEE7TRSJcBmJsp+XINvStp0frE2AGgnuUpFUObfOtHRb7oQcVsHtmYO6UWiMlpl04gtmRj8gHM/KDxhdW/wFZ1Sdea27w5AHDHjBGuZbKrzzzTVe/mc3MvzUuhSgQSXUldLfzW5jTs+rH0R/bskLBAERB3pdghazltuL0S/dPMURjUvX1C8FpPqiOlAGBgcXvccNoQEBHyQwGM6JO4noKewy7W0jbLaB75NGFgV4P1HjA9fESEyUN7xL7365p4j5uDTMYYHgHQAcD7pmGpkwCsIKLlAF4BcLUQQq4NeA2AvwOoQNSSeBuMJ8ge+6qt1dh3SB1EVQU45UPc1mbEhHy2VZNwZAA3mcRtQNTNYva1x9P9JG8xWAWazXTS+Xlj2TkDUiE5N8jJuDbsfP3mBiYcETETxs29tLo/quGqhS5n1bpxB7qZoSt/V72MPzi2n+X1ZJ0qiyHVSXepcLjBOZGf+XdTjXzSu2yDit8pzdRUnpCx8VFCiEEW5a8CeNViXzkA910uxjXSYpj+8ELLY1SjWGKuJJsXUL7AqsZI1uN23kTHNnl47JJxOHZAF0VdWgMdST4Zn3poqtD9G0W/NrZ8qeXL68ZiaJdEQ2U3e9ysZPW5klxZDDaNuHlIrFul6WY0kZtkb7IavaL5/TkjUbHzABZviq8fHYopBmuLIR1XUrLYzVWRmOfGmJ9782g+VcoMPySa9M/AWSajuGmYVdk4Y64km8ZDPsgq94Xc5zSDVC+n1RKjhhhDkr0qt7lm9Gky4vlupGJwPt9u7WwzdhaDuREOR0RMkbj5La169xEhEhSSl5k7XVkM2v/mBtCc8iNmMeTZuZKarwlzM0TbLKP5nIMmq0Nl/PlBMfjAaGEygTlA6WZCmzF/v9GVZOfLVfUAzfvCLoPPJx9pPcpMvjBCJP/yqLNTku7fxONiMYaYcktvtriZZOYTNEUi8dm4LuSwjDEIhcXgYYI2N9aHVLRm/7qZkMmVpA5sN6crKXnFYJ5kd9A0XFX1W7odwZdJWDG0UsxtjtMsUMDo2pDPpnwZ7Hy58kFWxhi0RrXBhftj8W8n45enH2l9QGx0UKJiUKWq1uO2V5xvcCVplzVZT16RTCLCcETEE7uFnc+zalwEEtdj8NZicB98VvnX9ci/QQa0VYfL5zIUIPzq9CHuBbXByg3nxup1So9ttiBU74UPDAZWDK0V88vvZrio4cHXnk75INsNVw0ogonxfVrdLq7fvajQtrcUtxgSg89O6ZxdK4ZQoitJ3stkZ3s7kcxEsya9YnBhMVg1uqrgs9sYgxvcWB+x58V0WXNeIWn1yDqVikF7Lnt3aoOeHdWzxpMlnZFOTn+/fIZkR6ZR0WFji4HJGKocLXbj3wHjxKeYxRCLMbhQDIoHWu5zO4/BDn1aB3Oj4tRTc+v711seEZNi0LsSnO6lG5LxTDVF4j19N7mOrGMMilFJHi4b6WZBGbc9Yjn82U6ptyuIyu7F8yWxG2jhhFMHRLqS5IRRldxOllRzwIqhlaJWDPbnqNYIlg+ynS/XroMjn3G38xjskENBiwpDiRZDCu6QI7Q8RV3b5+PaU44AEE0lIZHvrLwvB3WKIdUkhHqSyTQbdSVFt900glYxBkAoYgzeNQN6hXTveSPxn5+eqDhKp+FtMI9KUiGVmpeKwe56v5gyBH/5wVjrcx2sLxl8bmejGPyQVJoVQyvF3CtsCEccXReGpSBNriQ3FoOqF00WgdsHZ422lUXFacN64Nbpw/GbM4Ym9IhLOic/EehXU4/EM1eOx7j+XXDj1KHYdM9ZhnrNFsMh3YiSQ/X2bqXRfTs5Xj9pV5Imh5tGUDXZEIi6ksyxjXTXdFZx0fi+uPCYfhhZ0jFhH1npBVN7KH8LmZ+opJNqQl30uXQTQ3OLnSunMC9g2wlxciXJNZ3Ha8OxS7u1S+r6zQUrhlaKOU72YvkWx3HYqqUgX1+6FUA07YUVbuYxmC2GULLjTRF1VV154gC0zQ8lWCm/PG0I/jRzlO3s1XnXTTR8zwsGMGmIcRSU/qVctH4PDtSHlTGGMXe+n1D/8v85HQCw8rbT8eKPJzj+Pcl4o256bWUsb1U68xiEAOat2g4AePNnJ2LpLad53kPddM9Z+MO5R1vuv+L4UgBAUWEeTh1qPftZ/g3SWj1JMWKtg9bzPs8mz1Cy2LlyCkIB24EObq2vi4/tj/k3TMLEwYl/E7uSGFf85LkleHvl9qTOeeSDbxPKVlVW254TjkSwvfow9h9qSAgEti+wTt4ln+NeHQsTcs7Iep7QslZK+nZpgyuOL8WbP1O5GpwxN2ZtC0I4f1yJ7Us1rFcHx3r1imFhxW6U3fU+nv3iOwD2wxWf/9Gx6Ngmeo86FObZ9hy/uuU0RzlUyAR8O6qdk7nttUgR3iQEPloXzUjco6gwlsTv3vNG4tyxffDgrNF46KIxhsy2XnPtKYOw6Z6z0CY/iCcvL4un3tD0nUwtLa2BmWV98eLsCThzZK+EuvJDAXx9x1TcOn245fXcKo2TtE5CcQd11l0g+nxMGNgVfzh3JF6Ynaj8O7V1lxa7IBTAoO7q59EHeoEnuGUD81buwLyVOxJy19jxt082JpTlOfRmIgI47g8foE1eMMEF4MZiICJcfdIRuOfttbF9VlZxKBDAbVoK4UcvHuuYOz/hmmRWXHLkSrx8zhlDcc6YPuiiNX5y341TrYfEmnvadY0RbKiKrqdwyCYlwvFHdLPcZ6Zz2zz85OQjcNrwHs4HK1inWzDHKhHdOFPGWMn6qgOxbf2KYRce0w8XHpOYlsLM/ReMQi+PRv8Aan/6g7PGIBSgWC6mYIBw7MCulue3zbdvxv54/tHo1j4fj3+8AVeeMAD/92niuwEAN5x2JC4+tj/ue3edZV1njOyFwrwgLhrfD9/tOZiwX58rql1+0BCX0mMdA/KHK4kVQw7hNA4/NvqmsQkbdxsfetnwqrCbbGblptB7ks5Q9AQld31/RGyhGKtrPnl5Waz3Jd+pW6YPxw9PHJBwnpNytV9D2PZUWy4oK8FL5dHlRogIv3bILApEk8hV1dajtq4Rf/1ofcL+358zUpljCIimLH9x9gRc+IRx3Yylm/eje4cCHH9E15RGI9mlhk6X3587Eve+sxZlpZ1dT7rT/1y9OxYa9l136iCMLOmEYCCuPOwGUQQDhN6d2iif5ynDeuCG04YYnkX9cS/OnoCObfMMlnbbgpC1YrBxpfpBMbArKYeoc8gOqQ9KVtXWG/bZ9crsTF+rfWZXlRWXTOivTJGhr3fysHjPW75UndqklrferieXKpvuOQt/PN86tbUV548rwTUnH4FzxyrXq8L0UdYKFQAGaIHNn08ebCg/WB+2DE63JIO6t8ffLitLaia2vnE+flA3Qwrxn00eHLPKIqb0JvZ1JpZdelx/DO9tjF/pn8FjB3bF0J5FhkbdLt5gl8uKXUlMRli+Zb+y3Cn4/N7XOy332Y3EsPPrW1kT6XaKrHpV0kJJ9eXKZIN5/BFd0cdi6U478hSjhvKDARQ5LNrSvagQX91yGjq1ycOD/43HnA43Ntk2TNmE+fmaMLCLcl/ElBDRDtWzVah4/lXPdsitYrBzJflAM7BiaIXc+MpyZXmdw8zdf3y2yXKf3ZBGO1eSVfuT7rPvpHBSTUSWyQbz+R85j1RSoVTKLsXs0i4xGBoRQDADllFLYP6Z9S4a/U8p3aRuRuaq3J/mrKnR+hOPM1oM1pYPu5KYtEhlhq2Vm8ZN2mArVL1WiZ15bulKSlMzWF0ykKbF4IfMlmYyMc+gtVgM5kZU/13/jMk5I26eO9XtVi1ApLqFekvAbrKbncXAE9wYR1IJeFo9V+koBjtXkv3MZwuXT8qSaNe0uKh+hFQq+LHBVN77NCeS+6FX6gVmRW43fwNw93erOgeqxYxUz5gXMQY//DasGHxOMmkTJFaNomocftv8II7qbT0pTGI3qcfelWTl8knXYnCIMaRYrx9dLKmk+3DCD35sLzC3oVa/X8yV5OLvVj1bqt6/qv3W159v40qya/z98NuwYvA5qeRqs3rmVMNVv75jmu1i8BI7d4bdQ261J91n3+rl8XOMIVUy4UryowJMBXMnyOr3k4rB3agkdxaD6rnXxw7sniU716wP9AIHn/1OahaDutzKleTGdDX3WoniSsvuQc6UxWB1eroxhmTM+PPHleCVJZW2xwzp0T41QXRkwufsRwWYCgkWg83KdQAQJODmM4dhkM3voo4xuHQl6RSuXXzQ1mLwwW+TMYuBiG4joq1EtEz7nKnbdxMRVRDROiKaqisfR0QrtX0PkR+iMC1MsnrhqU83onLfYeU+q5QObhppcw9H34u1H67qWHVKWPX84hZDivUm8cjdd751PiAAWHHb6XhDmV205fHjPIZUSIwxqP+u+Kgkwo8mDcQpR1rnaFK6kpTDVRPPdatwbWMMPmj2Mv10PCCEGK195gEAEQ0HMAvAUQCmAfgrEUl1/CiA2QAGa59pGZbP1+yorsOS7/a5Pv5wQxNu/8/X2H/ImF7CvLaCGTc9lATFENIPC7SLPquL3Zj0dli7ktKrNxkLzanfUlSY5+laB16SrRbDgxeNNnw3/95OFoMrV5JqXRGbtUasri+fJKlUxvbrFNtn9+zIXWeOVK993hy0hCtpBoAXhBD1ADYSUQWA8US0CUCREGIRABDRMwC+D+DtFpDRF0z+80eWU+pV1IetG/5Ik7BMieH0rswcV5IQfC4IBSAz9qTiSkp7VJKDKynV9BWq5TYX3HgyfvzPJVi7ozZh3x/PPzrtvyUl0o3RZKliOHWoMb+U236AzPrupuPg9tYoFYNWVpgXiCVVlGnSpx7VE19t3u9YLxHhy5unxM5vCTJtMfyUiFYQ0f8RUWetrA+ALbpjKrWyPtq2uTwBIppNROVEVF5VVZUJuX2BWSmo/NlCCCzdHLUq5i7bpqwnIqIJvTbtOaTc7/Sy3DFjREIP5zpdmgW73o++bn1it/RjDBYWg/ZEpxKbAYDtpsylP588GP27tov1/Ad3N/qmLyjri5llfVO6VkuSrYrBjFvL85dTh+DsUb0xY3Rvx2PNK9zOHKfOD6V6BAMBws1nDsMbPz0RN585DCcO6hZL2ZLM+hvFHQoyMhrNLWlZDEQ0H4DK3rkZUbfQnYhaVHcC+DOAK6Hu6wib8sRCIZ4A8AQAlJWVpb80WJbwq5eX43zTQ/pyeSV+/eoKFBWGUFOnzv4ZEQJTR/TEa19tNZQP0ho5p0ZCTsZ5cNZo/PGdddi6/zAGdmtvmZDuxqlHxjJj6l+eM0b0jLnG0h6V5JASI1WL4XujeuOjb6rwg/H9MKZfJ8wYHe2b3DljBO57bx0e+cEYrN1eG1topbm4c8ZRuGXuas/qy1ZXkhmVS/F3Zw3DFxv3Gsq6dyjEQxeNcVWnfl2SDb8/0/I4q87NjyYNjG0/e9Wx+PN76/D+1ztdLcnqF9JSDEKIKW6OI6K/AXhT+1oJQN/FKgGwTSsvUZQzNqzfHU2jbKUUgGgAW5Vj/t9azn1HxaDtnzG6D14ur8TW/Ydte+TXnjLIcG2JvgeUfkoMdblsKFJdk/l7o3rje6MSe5UjSzrimSvHA0CzKwUAuPS4Ulx6XCnKN+3F+Y8tSvr88aVdsHhTvLFsLRaD6jm6auJAXDVxYOIOl+hXyLO3ht3VJwPiXi4/mmkyOSpJn/rxHACrtO03AMwiogIiGoBokHmxEGI7gFoimqCNRroMwNxMyddasHPJ6HuFqh6iXJDcya2jfzksl2W0QL+GgX4kk9vsqm5k0iP/TPPSpq2FohT9zi9dfRw+nXNq7HtrsRhS1P+2uH123CrXvFD0OC/WPW8uMhl8/iMRjUa0DdkE4McAIIRYTUQvAfgaQBjAtUII6Uy/BsA/ALRBNOics4Fnt9g9m4EAxXwqXiXtirtq3D3k+iGy+tmjmcquKn3OrVQvpNXT1w8gaC3DVTOB2wbc7Wj6vCy0GDKmGIQQl9rsuxvA3YrycgAjMiVTa8R2kRzt/5LObWx7iMm0NbFjXTa8B+r1FkN86Gamk+ilGnz2O+n09PUWW2uxGDJBU8TbBlwqZFYMTLNh18AKAE9dcQyG9y5KCDzrSWZOgTzSbcMr19EFTDEG11dU45R2O9UYg99Jx2LQn9taFacXhD02N+WSutnkSmJ7MstxaidOGdodPYoKPZtpmew8ge5FheinjVAqcDspLgk5rMqzqHOWFHYuQSc66Bb2yabea3PjdXwqG11JrBiyHNsGVvd8e7X4uFxOMpkcQLJ6w7jsDI1KiscYsqd3lgzpjia64vhSANnVe7XipCHFytF26SKHlf5iyhBP6pPB53AWKQZ2JWU5du2E0GkG+xiD9b4Vt51u+H7OmBKcNbJ3UpNvZGNtTKPh+nQllsFnrbi1KoZ0YwPZ6O+24mlt+LDXhLUYw5E900+ACOiHq2bPM8mKwae49ZHbxQf0jWcoxbTZqnWFk52RKV1VhuGqmZr5LF1drXRYUrrpsuVz4LUfvTUhLQavRm7JPGMNTRHMu24iGrJAKbNi8Cl26y/rsevtv3bNCbFtu8Y/0wNU5LX1smYs62qaM5/9jrQYUr19eYHWYzFkCqk07dyvySDnC7XLD2K4i0Wx/AArBp/ytGvFYL1P/xDarcCm6n3fd/7RmDCwqysZnFDNfUh3gpsVrX24arpBe9l7ZcVgjXQleTWk94RBXfG7s4Zh5rjsyanFwWcfsnTzPsuEd2bcNhTJmsUjSzrG8h2li/QgGRRDhiyG44+IKrNRfTtl5gItTH4wgK7t8nHX91Ob7nPaUdGEblOParmUzn5HZjVtm+9NynQiwlUTB6Jj25bLlposbDH4jHBTBE8u3Oj6eH1v/3dnDcNdb61RHpen6/3079oWndrm29br5QSodvmJj1mmFMOU4T2w4rbTlbGR1kAgQFhyy2kpnz+0Z5Fl8sNs4e+XlWF7tXoxKi/4/TkjMb60C8b265yxa/gdVgw+4/KnFuPTij2uj9e33xcf2x9nj+6N8Xf/N+E4fYzh5R8fh+5Fhbb1erl43oOzxuD5xZsxsk/HWFm6LhE7WqtSYKJMGd7D+aA06NQ2H1ecMCCj1/A7rBh8RjJKATA2sIV5AQQC6kZRvwJboQsT2cuGu2fHQtxwmnFMuFe1D+uVHcE8hskmWDG0IogIBSF1o6+3GAotjtGT6ZFKXiiej288BV3a27vEGIZJHlYMWY7b0Tf6oXd2I5QkmV49yguDpF9Xb4LjDMMY4VFJWY5VXpejSzoavutz7DjFD34zbSh6dWyTvnA2eBnDYBjGW9hiyHJUBkPF3WckuGqSybFzyYR+6YrFMEwWwxaDD/hq8z7DSmfJoHIlhYKBhFQZbtxHktay7CPDMKnBFkMLs+9gA87962eYelQPPHbJuKTPb3IZY0imsWfFwDDuePTisejR0X7odzbCFkOGOOPBT/By+RbH4w41Rpe+XFFZnVJiM7eZH/JskuiZSSfnP8PkEmeM7NUqJ8JxC+Axf35vHVZtrcaa7TW48ZUVjsfLLKAEoCGcfP4at1lEu7W3z1u/5o5psW02GBgmt8mYK4mIXgRwpPa1E4D9QojRRFQKYA2Addq+z4UQV2vnjAPwDwBtAMwD8HORRWs0NjZF8PAHFXhswfqkzlFtu0W6kt77xSTb47q0y8cDF46ytBza5Hu3HjPDMNlNxhSDEOJCuU1EfwZQrdu9XggxWnHaowBmA/gcUcUwDcDbmZLRa2Qe92QW5JC52RsjArsPNCR9TWkwDOnRwfHYc8aUJF0/wzC5R8ZdSRTtfl4A4F8Ox/UCUCSEWKRZCc8A+H6m5fOSxkhij3/T7oM4UG894ki6j6pq6zHl/gWO15i3crvheyQi2PXDMIynNEeMYSKAnUKIb3VlA4hoKREtIKKJWlkfAJW6Yyq1sgSIaDYRlRNReVVVVWakToG5S7cmlJ38p49w5T++tDwn2bjCT577yvA9IgSPImIYxlPSciUR0XwAqsTuNwsh5mrbF8FoLWwH0E8IsUeLKfybiI6COq+a0icjhHgCwBMAUFZW1uIxiEhE4GBDGLfMXa3cv3jjXstzrRRDgOxXITtQH8bjC9ajIRzJqpjAg7NG473VO1taDIZhbEhLMQghptjtJ6IQgHMBxAboCyHqAdRr20uIaD2AIYhaCHoneAmAbenI11zc//43eOTDCsv97WyymdZbBJyDAULEIlYRiQg88P43eHLhRnRum5dVrqQZo/tgxmilIcgwjE/ItCtpCoC1QoiYi4iIiokoqG0PBDAYwAYhxHYAtUQ0QYtLXAZgrqpSv/HaV5W2++WKUCrqG61dST8+aaCyvDESwYG6aNyisUkgmEUWA8Mw/ifTimEWEoPOkwCsIKLlAF4BcLUQQvpargHwdwAVANYjS0YkOc0+LrJRDA02Q1THl3ZRXy8iYtckZHbRG4Zhco+MpsQQQlyhKHsVwKsWx5cDSG0x2xbEafpBgU0Ka7vgs5W+aWwS8ayq5N2iNwzDMADPfPYEpzURzAnt9FgpBiEsIu/QLAbdjGkelcQwjJdwEj0PsFoTQWIXA2gINynLJw7uZqlwwk0Rg/vKK1fSwt+cgurDjZ7UxTBM9sKKwQOcLAa7vfUKi+HNn52II4rbY8E36jka4YiI5UgKRwQK8rxRDCWd26Kk9eUDYxgmSdiV5AFO2Zzs4ggHFbOih/Uq0nIXWVkMIpaJtT4cyarhqgzD+B9WDB7g5EqyUww12rDTm88cFiuTMQMrhROORGIWQ1NEuE69zTAM4wZWDCZWba1Gxa7apM5xGq5qNyT1QH0YvTsW4keTEucsWOmbsG64KgDsqq13JyjDMIwLWDGYmP7wQky5/+OkznFaE8HOYqita0SHwug8hwkDjfMWhIUraUPVAXy0zj85ohiGaV1w8NkDnCwGVYBZUlsXRofC6M/w3FUTDG4pq2qvfvYr9Q6GYRgPYIvBA5x8/PWN6iGpQFQxtNcUQzBAyNdNhnMa7cQwDJMJWDF4gNOooNr6MErnvIV/K9Jy19Y1on2B2nCbelRPTB7aHdeecgQumdDPC1EZhmEcYcWAaAzgT++uw6EG6wV17LCbeXz5cf1j29e/uAxb9h4y7D/Y0GSpGArzgnjyimNw49ShPPKIYZhmgxUDgBe/3IxHPqzAwx9Yp862w27m8YmDiw3fP9+wx/D9cEMT2uY7h3rs9MLQns7LejIMw7iFFQPiwWG7FNh22FkMbfKCmHPG0Nj3w7p4gxDRBX7aFViv16A/1op3rp/kUlKGYRhnWDE48Mm3VfjYIjWFJC9ofRsL8wKYMLBr7PvhhrhiqA9HIAS0Wc72KJaTBgBMP7qX47kMwzDJwMNVHbj0ycUAgE33nGV5jJ3FUBAKIi8U31+ns0pkOox2rlxJiRbDwt+cgpLObR3PZRiGSQa2GABUuZg5/M9Fm1B9SJ151C4lRkFeAIWhuEWgdyUd0qwHVxaD4hJt8pzPYxiGSZacVwx1jU14/OMNjsfdMnc17p73tXKf3cznglAAhboGvE6hGFxZDIpLFLJiYBgmA+S8YrDLY2RGNs61dY0onfMWnvviOwD2M58L84IozIvfZqNiiLqS2rqwGFTBZ1YMDMNkgpxXDMnQo6gQALCzpg4A8OTCjQAcXEkuLAY3ikHOgr7/glGxMl65jWGYTJCWYiCimUS0mogiRFRm2ncTEVUQ0ToimqorH0dEK7V9DxFFJwEQUQERvaiVf0FEpenIlgovL9kS264+3IiaOmNMobhDgbalNciaPrBLXVEQChrWfFbFGJKZx+DVam0MwzBWpDsqaRWAcwE8ri8kouEAZgE4CkBvAPOJaIgQognAowBmA/gcwDwA0wC8DeCHAPYJIQYR0SwA9wK4ME35kqK2Lj7zedTt7yXsl8NSyagXbC2G/FAAROpRSTFXkot5DPISRMDPJw/G6L6dHM9hGIZJhbQsBiHEGiHEOsWuGQBeEELUCyE2AqgAMJ6IegEoEkIsElGn+TMAvq8752lt+xUAk4n81T2ua2zCeY9+hhWV+2NlQgjLdROAuLunrH90zUy9xXCwPnlXEhHhF6cNwSlDuycrPsMwjCsyFWPoA2CL7nulVtZH2zaXG84RQoQBVAPoCgVENJuIyomovKqq+dYlWFG5H0u+24db566GJmeCUrBSZa9cczxOPrLYkGk1Hnx2Ybhp1+GwAsMwmcZRMRDRfCJapfjMsDtNUSZsyu3OSSwU4gkhRJkQoqy4uFh1iGuSSU53UIsJyJiBQHSZTT0hXctd2tU4+Sw/GDCszZBK8JmUt4lhGMY7HLuqQogpKdRbCaCv7nsJgG1aeYmiXH9OJRGFAHQEsDeFaydHEopBprPYfaAheqpITFURCgTQ2BQ97sNfnWzYlx8KGFZzO9TQhPxgwDalRkxMthgYhmkmMuVKegPALG2k0QAAgwEsFkJsB1BLRBO0+MFlAObqzrlc2z4fwAfCLnOcR1gtn6mi+nDizGfzHIZQMNpyt8kLwhwiKQgFtfxIAp+t343HFqw3zHGw4+i+HQGAU2AwDJNx0hqVRETnAHgYQDGAt4homRBiqhBiNRG9BOBrAGEA12ojkgDgGgD/ANAG0dFIb2vlTwL4JxFVIGopzEpHNrcko3r2HmwwfG+KCFz7nHGZTdn7715UADP5oQC27j+MATfNi5W5nYtw9aQjMGVYDwzpwSm2GYbJLGkpBiHE6wBet9h3N4C7FeXlAEYoyusAzExHnlRIxiTZd8ioGHbU1GHr/sMAorGFcETEYgz6/EgS/XwGidt5CYEAsVJgGKZZyOmZzxW7DmDsne+7Pv5Qg3HtZv38Bdnzl4pBn1FVolIMPhuRyzAMk7uKoSkiMOX+BZ7VJxVCSHMlhQLubi0HkxmG8Rs5qxjcpNpOhphC0ILPecHEFj+smAnHKS4YhvEbOasYvCbmQtIsBdUQVOl60s9bYIuBYRi/kbOKweuOunl0kUoxyMlwRYV5OjlYMzAM4y9yVjF4jbQY5LwGlStJWgwdCuODwVyGIhiGYZqNnG2WvO6nyxhDWFv4RxV8loqhqE3cYuAYA8MwfiNnFUOqXHnCAGW5tBgam6KNf8gm+FyktxhYMTAM4zNYMSTJoO7tleUyxiCtgnxFjOGak47AgG7tMLMsnkaqf1dOccEwjL9Id6GenMMq351UDL06FWJHTR2G9y5KOGZwjw748Fcno/pQPOfSgxeOyYicDMMwqZKziiHV7HxWrh/pOjpxUDfcMn04Rpd0sqyjY9s8dGqbh/PHlqBj2zzL4xiGYVqC3FUMKWoGq6R3+mDz2H6dHetZduvpqQnAMAyTYXI2xhBJUTNYKQbV8FSGYZhsJGcVg9euJB5dxDBMayFnFUNEkbfIDVYWQ5d2+QBSd1ExDMP4hZxVDKliZRmUdmvXzJIwDMNkhpxVDKnGGKyS3rXNS1yYh2EYJhvJWcXg5aik//z0xDSlYRiG8Q+5qxhSPM+sGIiAkSUddfVykIFhmOwmLcVARDOJaDURRYioTFd+GhEtIaKV2v+n6vZ9RETriGiZ9umulRcQ0YtEVEFEXxBRaTqyOeHVcFUZc+BBSQzDtBbSneC2CsC5AB43le8G8D0hxDYiGgHgXQB9dPsvFkKUm875IYB9QohBRDQLwL0ALkxTPktSdSWZg89ST/BoJIZhWgtpWQxCiDVCiHWK8qVCiG3a19UAComowKG6GQCe1rZfATCZMriKjUg5+Gx2JZm+e57Qm2EYpnlpjhjDeQCWCiH0iyw/pbmRbtE1/n0AbAEAIUQYQDWArqoKiWg2EZUTUXlVVVVKQnkVYwiaFAPHGBiGyXYcFQMRzSeiVYrPDBfnHoWoS+jHuuKLhRAjAUzUPpfKwxVVKFtZIcQTQogyIURZcXGxkxhKUo8xGL9LPcExBoZhWguOMQYhxJRUKiaiEgCvA7hMCLFeV99W7f9aInoewHgAzwCoBNAXQCURhQB0BLA3lWu7wbsYA2sEhmFaFxlxJRFRJwBvAbhJCPGprjxERN207TwA0xENYAPAGwAu17bPB/CBSDUQ4AKv5jGwXmAYprWR7nDVc4ioEsBxAN4ione1XT8FMAjALaZhqQUA3iWiFQCWAdgK4G/aOU8C6EpEFQBuADAnHdmcSH3ms8liMCkKHp3EMEy2k9ZwVSHE64i6i8zldwG4y+K0cRZ11QGYmY48zYH1PAY2HRiGaR3k7Mxn7ya4eSENwzCMf8hZxZB68Nn4nS0FhmFaGzmrGFK3GIy3zKwoOMTAMEy2k7OKIeUJbjxclWGYVk7uKoYUNYNZD7BiYBimtZHDiiHF4arm4HPO3kGGYVorOduspepKMscUzBYDz2NgGCbbyVnFEImk1oKbs6fyegwMw7Q2clYxpNqxNysAVggMw7Q2clcxpBp8Nn3n4DPDMK2NHFYM9prhhtOGKMvNE9p45jPDMK2N3FUMDvvNqS8kTsNVeaEehmGynZxVDE4zny0Vg/m7DD7zkp4Mw7QSclYxOMUYQpYWg3mNZ4ZhmNZFzioGJ4vBKqjMMQWGYVo7OasYnCIBoaCVK8lhBTcOMTAMk+XkrGJwasAth6FaFbMlwTBMKyFnFYOTK8k6xmD/nWEYJtvJWcXgFHy2GpXEE9oYhmntpKUYiGgmEa0moggRlenKS4noMBEt0z6P6faNI6KVRFRBRA+RNsyHiAqI6EWt/AsiKk1HNic8G64K8zwGhmGY7CZdi2EVgHMBfKzYt14IMVr7XK0rfxTAbACDtc80rfyHAPYJIQYBeADAvWnKZotXE9xi5emJwzAM4xvSUgxCiDVCiHVujyeiXgCKhBCLRDQnxTMAvq/tngHgaW37FQCTKYMLKjulxLC2GBxGJTEMw2Q5mYwxDCCipUS0gIgmamV9AFTqjqnUyuS+LQAghAgDqAbQVVUxEc0monIiKq+qqkpJOKcYQ/cOhcryhOBzSldnGIbxLyGnA4hoPoCeil03CyHmWpy2HUA/IcQeIhoH4N9EdBTU7ahsou32GQuFeALAEwBQVlaWklvf7qR3rp+Ibu0LlPsSLISEhXo4ysAwTHbjqBiEEFOSrVQIUQ+gXtteQkTrAQxB1EIo0R1aAmCbtl0JoC+ASiIKAegIYG+y13aLVfD5tZ8cj6E9i7DvYINyv1VOJHYpMQzTWsiIK4mIiokoqG0PRDTIvEEIsR1ALRFN0OIHlwGQVscbAC7Xts8H8IHIYPfbquax/ToDSFzbWcKuJIZhWjvpDlc9h4gqARwH4C0ielfbNQnACiJajmgg+WohhOz9XwPg7wAqAKwH8LZW/iSArkRUAeAGAHPSkc0Jr4arMgzDtDYcXUl2CCFeB/C6ovxVAK9anFMOYISivA7AzHTk8ZKgZRI9+1FJHGJgGCbbydmZz47ZVS3ujPU8BrYlGIZpHeSsYnDq2VulvuD1GBiGae2wYrDAypXEMAzT2slZxeDsSnKnGMwWBIcYGIbJdnJWMXjVgEu1wAYGwzCthdxVDDx8iGEYRkkOKwZv6nHrcmIYhskWclYxRFwohjV3TMOUYd1tj8kLmnMlpSMVwzBMy5OzikG4iDK0yQ+iazt1Mj1J0GrCA8MwTJaSs62a2579Ld8bjtu+N9xyv9Xa0AzDMNlKDisGd5qhfUEIV5wwwHK/VU4lhmGYbCWtXEnZTLKhgLeuOxGNTYlnmWMMDMMw2U7OKoaIm+izjqN6d1SWyxjDhcf0xVeb9+HaU45IWzaGYZiWJGcVg1eDh2SMoUNhHv568TiPamUYhmk5cjbGkKTBYAnHGBiGaW3krGLwauYzxxgYhmlt5Kxi8Aq2GBiGaW3krGJwyq7qlhBPcGMYppWRs62aV6kr2GJgGKa1kZZiIKKZRLSaiCJEVKYrv5iIluk+ESIare37iIjW6fZ118oLiOhFIqogoi+IqDQd2ZwY1qsIFx/bL+16QhxjYBimlZGuxbAKwLkAPtYXCiGeE0KMFkKMBnApgE1CiGW6Qy6W+4UQu7SyHwLYJ4QYBOABAPemKZstk4YU4+5zRqZdD6fEYBimtZGWYhBCrBFCrHM47CIA/3JR3QwAT2vbrwCYTObl0XwIJ9FjGKa10Ryt2oVIVAxPaW6kW3SNfx8AWwBACBEGUA2gq6pCIppNROVEVF5VVZUpuV3BFgPDMK0NR8VARPOJaJXiM8PFuccCOCSEWKUrvlgIMRLARO1zqTxcUYUyRCyEeEIIUSaEKCsuLnYSw5YrbRLkuaFHkX1aboZhmGzDMSWGEGJKGvXPgslaEEJs1f6vJaLnAYwH8AyASgB9AVQSUQhARwB707i2K26ZPgxnHd0LX323D6c6LMojufe8kSjt2g47aurwvaN7Z1hChmGY5iVjuZKIKABgJoBJurIQgE5CiN1ElAdgOoD52u43AFwOYBGA8wF8IJphYWYiwrj+nTGuf2fX51x4TPqjmRiGYfxKWoqBiM4B8DCAYgBvEdEyIcRUbfckAJVCiA26UwoAvKsphSCiSuFv2r4nAfyTiCoQtRRmpSMbwzAMkxrUDJ3yjFJWVibKy8tbWgyGYZisgoiWCCHKVPt4rCXDMAxjgBUDwzAMY4AVA8MwDGOAFQPDMAxjgBUDwzAMY4AVA8MwDGMg64erElEVgO9SPL0bgN0eipMJ/C6j3+UD/C+j3+UD/C+j3+UD/CdjfyGEMqdQ1iuGdCCicqtxvH7B7zL6XT7A/zL6XT7A/zL6XT4gO2SUsCuJYRiGMcCKgWEYhjGQ64rhiZYWwAV+l9Hv8gH+l9Hv8gH+l9Hv8gHZISOAHI8xMAzDMInkusXAMAzDmGDFwDAMwxjIWcVARNOIaB0RVRDRnBaS4f+IaBcRrdKVdSGi94noW+3/zrp9N2nyriOiqepaPZWvLxF9SERriGg1Ef3chzIWEtFiIlquyXi732TUrhkkoqVE9KZP5dtERCu1tdjL/SYjEXUioleIaK32PB7nM/mO1O6d/NQQ0fV+kjEphBA590F0kaD1AAYCyAewHMDwFpBjEoCxAFbpyv4IYI62PQfAvdr2cE3OAgADNPmDGZavF4Cx2nYHAN9ocvhJRgLQXtvOA/AFgAl+klG77g0Angfwpt9+Z+26mwB0M5X5RkYATwO4StvOB9DJT/KZZA0C2AGgv19ldPwbWlqAFvmjgeMAvKv7fhOAm1pIllIYFcM6AL207V4A1qlkBPAugOOaWda5AE7zq4wA2gL4CsCxfpIRQAmA/wI4VacYfCOfdh2VYvCFjACKAGyENljGb/Ip5D0dwKd+ltHpk6uupD4Atui+V2plfqCHEGI7AGj/d9fKW1RmIioFMAbRHrmvZNTcNMsA7ALwvhDCbzL+L4BfA4joyvwkHwAIAO8R0RIimu0zGQcCqALwlOaO+zsRtfORfGZmAfiXtu1XGW3JVcVAijK/j9ttMZmJqD2AVwFcL4SosTtUUZZxGYUQTUKI0Yj2zMcT0Qibw5tVRiKaDmCXEGKJ21MUZc3xO58ghBgL4AwA1xLRJJtjm1vGEKIu10eFEGMAHETULWNFS74r+QDOBvCy06GKMt+0QbmqGCoB9NV9LwGwrYVkMbOTiHoBgPb/Lq28RWQmojxElcJzQojX/CijRAixH8BHAKb5SMYTAJxNRJsAvADgVCJ61kfyAQCEENu0/3cBeB3AeB/JWAmgUrMEAeAVRBWFX+TTcwaAr4QQO7XvfpTRkVxVDF8CGExEAzQNPwvAGy0sk+QNAJdr25cj6teX5bOIqICIBgAYDGBxJgUhIgLwJIA1Qoj7fSpjMRF10rbbAJgCYK1fZBRC3CSEKBFClCL6nH0ghLjEL/IBABG1I6IOchtRH/kqv8gohNgBYAsRHakVTQbwtV/kM3ER4m4kKYvfZHSmpYMcLfUBcCaio2zWA7i5hWT4F4DtABoR7UH8EEBXRAOV32r/d9Edf7Mm7zoAZzSDfCciat6uALBM+5zpMxmPBrBUk3EVgFu1ct/IqLvuyYgHn30jH6I+/OXaZ7V8H3wm42gA5drv/G8Anf0kn3bNtgD2AOioK/OVjG4/nBKDYRiGMZCrriSGYRjGAlYMDMMwjAFWDAzDMIwBVgwMwzCMAVYMDMMwjAFWDAzDMIwBVgwMwzCMgf8P0FhxoDLPFPIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def play_episode(env, agent, max_episode_steps=None, mode=None, render=False):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    agent.reset(mode=mode)\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    while True:\n",
    "        action = agent.step(observation, reward, done)\n",
    "        if render:\n",
    "            env.render()\n",
    "        if done:\n",
    "            break\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if max_episode_steps and elapsed_steps >= max_episode_steps:\n",
    "            break\n",
    "    agent.close()\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "\n",
    "logging.info('==== train ====')\n",
    "episode_rewards = []\n",
    "for episode in itertools.count():\n",
    "    episode_reward, elapsed_steps = play_episode(env.unwrapped, agent,\n",
    "            max_episode_steps=env._max_episode_steps, mode='train')\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('train episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "    if np.mean(episode_rewards[-10:]) > -120:\n",
    "        break\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "\n",
    "logging.info('==== test ====')\n",
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
